You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ts...@apache.org on 2013/06/17 09:04:11 UTC

[2/3] Moving maintenance mode into component/maint

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0587d3a4/test/integration/component/maint/test_vpc_host_maintenance.py
----------------------------------------------------------------------
diff --git a/test/integration/component/maint/test_vpc_host_maintenance.py b/test/integration/component/maint/test_vpc_host_maintenance.py
new file mode 100644
index 0000000..8fc427a
--- /dev/null
+++ b/test/integration/component/maint/test_vpc_host_maintenance.py
@@ -0,0 +1,561 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+""" Component tests VM life cycle in VPC network functionality
+"""
+#Import Local Modules
+import marvin
+from nose.plugins.attrib import attr
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.integration.lib.utils import *
+from marvin.integration.lib.base import *
+from marvin.integration.lib.common import *
+from marvin.remoteSSHClient import remoteSSHClient
+import datetime
+
+
+class Services:
+    """Test VM life cycle in VPC network services
+    """
+
+    def __init__(self):
+        self.services = {
+                         "account": {
+                                    "email": "test@test.com",
+                                    "firstname": "Test",
+                                    "lastname": "User",
+                                    "username": "test",
+                                    # Random characters are appended for unique
+                                    # username
+                                    "password": "password",
+                                    },
+                         "service_offering": {
+                                    "name": "Tiny Instance",
+                                    "displaytext": "Tiny Instance",
+                                    "cpunumber": 1,
+                                    "cpuspeed": 100,
+                                    "memory": 64,
+                                    },
+                        "service_offering_1": {
+                                    "name": "Tiny Instance- tagged host 1",
+                                    "displaytext": "Tiny off-tagged host2",
+                                    "cpunumber": 1,
+                                    "cpuspeed": 100,
+                                    "memory": 64,
+                                    "tags": "HOST_TAGS_HERE"
+                                    },
+                         "service_offering_2": {
+                                    "name": "Tiny Instance- tagged host 2",
+                                    "displaytext": "Tiny off-tagged host2",
+                                    "cpunumber": 1,
+                                    "cpuspeed": 100,
+                                    "memory": 64,
+                                    "tags": "HOST_TAGS_HERE"
+                                    },
+                         "network_offering": {
+                                    "name": 'VPC Network offering',
+                                    "displaytext": 'VPC Network off',
+                                    "guestiptype": 'Isolated',
+                                    "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL',
+                                    "traffictype": 'GUEST',
+                                    "availability": 'Optional',
+                                    "useVpc": 'on',
+                                    "serviceProviderList": {
+                                            "Dhcp": 'VpcVirtualRouter',
+                                            "Dns": 'VpcVirtualRouter',
+                                            "SourceNat": 'VpcVirtualRouter',
+                                            "PortForwarding": 'VpcVirtualRouter',
+                                            "Lb": 'VpcVirtualRouter',
+                                            "UserData": 'VpcVirtualRouter',
+                                            "StaticNat": 'VpcVirtualRouter',
+                                            "NetworkACL": 'VpcVirtualRouter'
+                                        },
+                                },
+                         "network_offering_no_lb": {
+                                    "name": 'VPC Network offering',
+                                    "displaytext": 'VPC Network off',
+                                    "guestiptype": 'Isolated',
+                                    "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,UserData,StaticNat,NetworkACL',
+                                    "traffictype": 'GUEST',
+                                    "availability": 'Optional',
+                                    "useVpc": 'on',
+                                    "serviceProviderList": {
+                                            "Dhcp": 'VpcVirtualRouter',
+                                            "Dns": 'VpcVirtualRouter',
+                                            "SourceNat": 'VpcVirtualRouter',
+                                            "PortForwarding": 'VpcVirtualRouter',
+                                            "UserData": 'VpcVirtualRouter',
+                                            "StaticNat": 'VpcVirtualRouter',
+                                            "NetworkACL": 'VpcVirtualRouter'
+                                        },
+                                },
+                         "network_off_shared": {
+                                    "name": 'Shared Network offering',
+                                    "displaytext": 'Shared Network offering',
+                                    "guestiptype": 'Shared',
+                                    "traffictype": 'GUEST',
+                                    "availability": 'Optional',
+                                    "useVpc": 'on',
+                                    "specifyIpRanges": True,
+                                    "specifyVlan": True
+                                    },
+                         "vpc_offering": {
+                                    "name": 'VPC off',
+                                    "displaytext": 'VPC off',
+                                    "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat',
+                                },
+                         "vpc": {
+                                 "name": "TestVPC",
+                                 "displaytext": "TestVPC",
+                                 "cidr": '10.0.0.1/24'
+                                 },
+                         "network": {
+                                  "name": "Test Network",
+                                  "displaytext": "Test Network",
+                                  "netmask": '255.255.255.0',
+                                  "limit": 5,
+                                  # Max networks allowed as per hypervisor
+                                  # Xenserver -> 5, VMWare -> 9
+                                },
+                         "lbrule": {
+                                    "name": "SSH",
+                                    "alg": "leastconn",
+                                    # Algorithm used for load balancing
+                                    "privateport": 22,
+                                    "publicport": 2222,
+                                    "openfirewall": False,
+                                    "startport": 2222,
+                                    "endport": 2222,
+                                    "protocol": "TCP",
+                                    "cidrlist": '0.0.0.0/0',
+                                },
+                         "natrule": {
+                                    "privateport": 22,
+                                    "publicport": 22,
+                                    "startport": 22,
+                                    "endport": 22,
+                                    "protocol": "TCP",
+                                    "cidrlist": '0.0.0.0/0',
+                                },
+                         "fw_rule": {
+                                    "startport": 1,
+                                    "endport": 6000,
+                                    "cidr": '0.0.0.0/0',
+                                    # Any network (For creating FW rule)
+                                    "protocol": "TCP"
+                                },
+                         "http_rule": {
+                                    "startport": 80,
+                                    "endport": 80,
+                                    "cidrlist": '0.0.0.0/0',
+                                    "protocol": "ICMP"
+                                },
+                         "virtual_machine": {
+                                    "displayname": "Test VM",
+                                    "username": "root",
+                                    "password": "password",
+                                    "ssh_port": 22,
+                                    "hypervisor": 'XenServer',
+                                    # Hypervisor type should be same as
+                                    # hypervisor type of cluster
+                                    "privateport": 22,
+                                    "publicport": 22,
+                                    "protocol": 'TCP',
+                                    "userdata": 'This is sample data',
+                                },
+                         "ostype": 'CentOS 5.3 (64-bit)',
+                         # Cent OS 5.3 (64 bit)
+                         "sleep": 60,
+                         "timeout": 10,
+                         "mode": 'advanced'
+                    }
+
+
+class TestVMLifeCycleHostmaintenance(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.api_client = super(
+                               TestVMLifeCycleHostmaintenance,
+                               cls
+                               ).getClsTestClient().getApiClient()
+        cls.services = Services().services
+        # Get Zone, Domain and templates
+        cls.domain = get_domain(cls.api_client, cls.services)
+        cls.zone = get_zone(cls.api_client, cls.services)
+        cls.template = get_template(
+                            cls.api_client,
+                            cls.zone.id,
+                            cls.services["ostype"]
+                            )
+        cls.services["virtual_machine"]["zoneid"] = cls.zone.id
+        cls.services["virtual_machine"]["template"] = cls.template.id
+
+        cls.service_offering_1 = ServiceOffering.create(
+                                            cls.api_client,
+                                            cls.services["service_offering_1"]
+                                            )
+        cls.service_offering_2 = ServiceOffering.create(
+                                            cls.api_client,
+                                            cls.services["service_offering_2"]
+                                            )
+        cls.vpc_off = VpcOffering.create(
+                                     cls.api_client,
+                                     cls.services["vpc_offering"]
+                                     )
+        cls.vpc_off.update(cls.api_client, state='Enabled')
+
+        cls.account = Account.create(
+                                     cls.api_client,
+                                     cls.services["account"],
+                                     admin=True,
+                                     domainid=cls.domain.id
+                                     )
+
+        cls.vpc_off = VpcOffering.create(
+                                     cls.api_client,
+                                     cls.services["vpc_offering"]
+                                     )
+
+        cls.vpc_off.update(cls.api_client, state='Enabled')
+
+        cls.services["vpc"]["cidr"] = '10.1.1.1/16'
+        cls.vpc = VPC.create(
+                         cls.api_client,
+                         cls.services["vpc"],
+                         vpcofferingid=cls.vpc_off.id,
+                         zoneid=cls.zone.id,
+                         account=cls.account.name,
+                         domainid=cls.account.domainid
+                         )
+
+        cls.nw_off = NetworkOffering.create(
+                                            cls.api_client,
+                                            cls.services["network_offering"],
+                                            conservemode=False
+                                            )
+        # Enable Network offering
+        cls.nw_off.update(cls.api_client, state='Enabled')
+
+        # Creating network using the network offering created
+        cls.network_1 = Network.create(
+                                cls.api_client,
+                                cls.services["network"],
+                                accountid=cls.account.name,
+                                domainid=cls.account.domainid,
+                                networkofferingid=cls.nw_off.id,
+                                zoneid=cls.zone.id,
+                                gateway='10.1.1.1',
+                                vpcid=cls.vpc.id
+                                )
+        cls.nw_off_no_lb = NetworkOffering.create(
+                                    cls.api_client,
+                                    cls.services["network_offering_no_lb"],
+                                    conservemode=False
+                                    )
+        # Enable Network offering
+        cls.nw_off_no_lb.update(cls.api_client, state='Enabled')
+
+        # Creating network using the network offering created
+        cls.network_2 = Network.create(
+                                cls.api_client,
+                                cls.services["network"],
+                                accountid=cls.account.name,
+                                domainid=cls.account.domainid,
+                                networkofferingid=cls.nw_off_no_lb.id,
+                                zoneid=cls.zone.id,
+                                gateway='10.1.2.1',
+                                vpcid=cls.vpc.id
+                                )
+        # Spawn an instance in that network
+        cls.vm_1 = VirtualMachine.create(
+                                  cls.api_client,
+                                  cls.services["virtual_machine"],
+                                  accountid=cls.account.name,
+                                  domainid=cls.account.domainid,
+                                  serviceofferingid=cls.service_offering_1.id,
+                                  networkids=[str(cls.network_1.id)]
+                                  )
+        # Spawn an instance in that network
+        cls.vm_2 = VirtualMachine.create(
+                                  cls.api_client,
+                                  cls.services["virtual_machine"],
+                                  accountid=cls.account.name,
+                                  domainid=cls.account.domainid,
+                                  serviceofferingid=cls.service_offering_1.id,
+                                  networkids=[str(cls.network_1.id)]
+                                  )
+        cls.vm_3 = VirtualMachine.create(
+                                  cls.api_client,
+                                  cls.services["virtual_machine"],
+                                  accountid=cls.account.name,
+                                  domainid=cls.account.domainid,
+                                  serviceofferingid=cls.service_offering_2.id,
+                                  networkids=[str(cls.network_2.id)]
+                                  )
+        routers = Router.list(
+                              cls.api_client,
+                              account=cls.account.name,
+                              domainid=cls.account.domainid,
+                              listall=True
+                              )
+        if isinstance(routers, list):
+            cls.vpcvr = routers[0]
+
+        cls._cleanup = [
+                        cls.service_offering_1,
+                        cls.service_offering_2,
+                        cls.nw_off,
+                        cls.nw_off_no_lb,
+                        ]
+        return
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cls.account.delete(cls.api_client)
+            wait_for_cleanup(cls.api_client, ["account.cleanup.interval"])
+            #Cleanup resources used
+            cleanup_resources(cls.api_client, cls._cleanup)
+
+            # Waiting for network cleanup to delete vpc offering
+            wait_for_cleanup(cls.api_client, ["network.gc.wait",
+                                              "network.gc.interval"])
+            cls.vpc_off.delete(cls.api_client)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(self):
+
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self.debug("Check the status of VPC virtual router")
+        routers = Router.list(
+                              self.apiclient,
+                              networkid=self.network_1.id,
+                              listall=True
+                              )
+        if not isinstance(routers, list):
+            raise Exception("No response from list routers API")
+
+        self.router = routers[0]
+        if self.router.state == "Running":
+            Router.stop(self.apiclient, id=self.router.id)
+
+        self.cleanup = []
+        return
+
+    def tearDown(self):
+        try:
+            #Clean up, terminate the created network offerings
+            cleanup_resources(self.apiclient, self.cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def validate_vm_deployment(self):
+        """Validates VM deployment on different hosts"""
+
+        vms = VirtualMachine.list(
+                                  self.apiclient,
+                                  account=self.account.name,
+                                  domainid=self.account.domainid,
+                                  networkid=self.network_1.id,
+                                  listall=True
+                                  )
+        self.assertEqual(
+                         isinstance(vms, list),
+                         True,
+                         "List VMs shall return a valid response"
+                         )
+        host_1 = vms[0].hostid
+        self.debug("Host for network 1: %s" % vms[0].hostid)
+
+        vms = VirtualMachine.list(
+                                  self.apiclient,
+                                  account=self.account.name,
+                                  domainid=self.account.domainid,
+                                  networkid=self.network_2.id,
+                                  listall=True
+                                  )
+        self.assertEqual(
+                         isinstance(vms, list),
+                         True,
+                         "List VMs shall return a valid response"
+                         )
+        host_2 = vms[0].hostid
+        self.debug("Host for network 2: %s" % vms[0].hostid)
+
+        self.assertNotEqual(
+                host_1,
+                host_2,
+                "Both the virtual machines should be deployed on diff hosts "
+                )
+        return
+
+    @attr(tags=["advanced", "intervlan"])
+    def test_01_enable_maintenance_with_vpc_nw(self):
+        """ Test enable Maintenance Mode on Hosts which have VPC elements
+        """
+
+        # Validate the following
+        # 1. Create a VPC with cidr - 10.1.1.1/16
+        # 2. Add network1(10.1.1.1/24) and network2(10.1.2.1/24) to this VPC.
+        # 3. Deploy vm1 and vm2 in network1 and vm3 and vm4 in network2. Make
+        #    sure vm1 and vm3 are deployed on one host in the cluster while
+        #    vm2 and vm4 are deployed on the other host in the cluster. This
+        #    can be done using host's tags & service offerings with host tags
+        # Steps:
+        # 1.Enable Maintenance on one of host on which VPCVR is present
+        # Validations:
+        # 1. Successfully push the host into maintenance mode.
+        # 2. VMs present on the above host should successfully migrate to the
+        #    other host present in the cluster
+
+        self.validate_vm_deployment()
+        self.debug("Stop the host on which the VPC virtual router is running")
+        try:
+            Host.enableMaintenance(self.apiclient, id=self.vpcvr.hostid)
+        except Exception as e:
+            self.fail("Failed to enable maintenance mode on host: %s" % e)
+
+        self.debug(
+            "Check if all instances belonging to the account are up again?")
+        routers = Router.list(
+                              self.apiclient,
+                              account=self.account.name,
+                              domainid=self.account.domainid,
+                              listall=True
+                              )
+        self.assertEqual(
+                         isinstance(routers, list),
+                         True,
+                         "List routers shall return a valid VPCVR for account"
+                         )
+        for router in routers:
+            self.assertEqual(
+                             router.state,
+                             "Running",
+                             "Router state should be running after migration"
+                             )
+        vms = VirtualMachine.list(
+                                  self.apiclient,
+                                  account=self.account.name,
+                                  domainid=self.account.domainid,
+                                  listall=True
+                                  )
+        self.assertEqual(
+                    isinstance(vms, list),
+                    True,
+                    "VM response should return instances running for account"
+                    )
+        for vm in vms:
+            self.assertEqual(
+                             vm.state,
+                             "Ruuning",
+                             "Vm state should be running after migration"
+                             )
+        return
+
+    @attr(tags=["advanced", "intervlan"])
+    def test_02_cancel_maintenance(self):
+        """ Test cancel Maintenance Mode on the above Hosts + Migrate VMs Back
+        """
+
+        # Steps
+        # 1. Cancel Maintenance Mode on the host.
+        # 2. Migrate the VMs back onto the host on which Maintenance mode is
+        #    cancelled.
+        # Validate the following
+        # 1. Successfully cancel the Maintenance mode on the host.
+        # 2. Migrate the VMs back successfully onto the host.
+        # 3. Check that the network connectivity exists with the migrated VMs.
+
+        self.debug("Cancel host maintenence on which the VPCVR is running")
+        try:
+            Host.cancelMaintenance(self.apiclient, id=self.vpcvr.hostid)
+        except Exception as e:
+            self.fail("Failed to enable maintenance mode on host: %s" % e)
+
+        self.debug(
+            "Migrating the instances back to the host: %s" %
+                                                        self.vpcvr.hostid)
+        try:
+            cmd = migrateSystemVm.migrateSystemVmCmd()
+            cmd.hostid = self.vpcvr.hostid
+            cmd.virtualmachineid = self.vpcvr.id
+            self.apiclient.migrateSystemVm(cmd)
+        except Exception as e:
+            self.fail("Failed to migrate VPCVR back: %s" % e)
+
+        self.debug("Check the status of router after migration")
+        routers = Router.list(
+                              self.apiclient,
+                              id=self.vpcvr.id,
+                              listall=True
+                              )
+        self.assertEqual(
+                         isinstance(routers, list),
+                         True,
+                         "List routers shall return the valid response"
+                         )
+        self.assertEqual(
+                         routers[0].state,
+                         "Running",
+                         "Router state should be running"
+                         )
+        #  TODO: Check for the network connectivity
+        return
+
+    @attr(tags=["advanced", "intervlan"])
+    def test_03_reconnect_host(self):
+        """ Test reconnect Host which has VPC elements
+        """
+
+        # Steps:
+        # 1.Reconnect one of the host on which VPC Virtual Router is present.
+        # Validate the following
+        # 1. Host should successfully reconnect.
+        # 2. Network connectivity to all the VMs on the host should not be
+        #    effected due to reconnection.
+
+        self.debug("Reconnecting the host where VPC VR is running")
+        try:
+            Host.reconnect(self.apiclient, id=self.vpcvr.hostid)
+        except Exception as e:
+            self.fail("Failed to reconnect to host: %s" % e)
+
+        self.debug("Check the status of router after migration")
+        routers = Router.list(
+                              self.apiclient,
+                              id=self.vpcvr.id,
+                              listall=True
+                              )
+        self.assertEqual(
+                         isinstance(routers, list),
+                         True,
+                         "List routers shall return the valid response"
+                         )
+        self.assertEqual(
+                         routers[0].state,
+                         "Running",
+                         "Router state should be running"
+                         )
+        #  TODO: Check for the network connectivity
+        return
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0587d3a4/test/integration/component/maint/test_vpc_on_host_maintenance.py
----------------------------------------------------------------------
diff --git a/test/integration/component/maint/test_vpc_on_host_maintenance.py b/test/integration/component/maint/test_vpc_on_host_maintenance.py
new file mode 100644
index 0000000..6630ee6
--- /dev/null
+++ b/test/integration/component/maint/test_vpc_on_host_maintenance.py
@@ -0,0 +1,323 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from nose.plugins.attrib import attr
+from marvin.cloudstackTestCase import *
+from marvin.cloudstackAPI import *
+from marvin.integration.lib.utils import *
+from marvin.integration.lib.base import *
+from marvin.integration.lib.common import *
+
+
+class Services:
+    """Test VPC services
+    """
+
+    def __init__(self):
+        self.services = {
+                         "account": {
+                                    "email": "test@test.com",
+                                    "firstname": "Test",
+                                    "lastname": "User",
+                                    "username": "test",
+                                    # Random characters are appended for unique
+                                    # username
+                                    "password": "password",
+                                    },
+                          "service_offering": {
+                                    "name": "Tiny Instance",
+                                    "displaytext": "Tiny Instance",
+                                    "cpunumber": 1,
+                                    "cpuspeed": 100,
+                                    "memory": 128,
+                                    },
+                         "vpc_offering": {
+                                    "name": 'VPC off',
+                                    "displaytext": 'VPC off',
+                                    "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat,NetworkACL',
+                                },
+                         "vpc": {
+                                 "name": "TestVPC",
+                                 "displaytext": "TestVPC",
+                                 "cidr": '10.0.0.1/24'
+                                 },
+                         "virtual_machine": {
+                                    "displayname": "Test VM",
+                                    "username": "root",
+                                    "password": "password",
+                                    "ssh_port": 22,
+                                    "hypervisor": 'XenServer',
+                                    # Hypervisor type should be same as
+                                    # hypervisor type of cluster
+                                    "privateport": 22,
+                                    "publicport": 22,
+                                    "protocol": 'TCP',
+                                },
+                         "ostype": 'CentOS 5.3 (64-bit)',
+                         # Cent OS 5.3 (64 bit)
+                         "sleep": 60,
+                         "timeout": 10
+                    }
+
+class TestVPCHostMaintenance(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.api_client = super(
+            TestVPCHostMaintenance,
+            cls
+        ).getClsTestClient().getApiClient()
+        cls.services = Services().services
+        # Get Zone, Domain and templates
+        cls.domain = get_domain(cls.api_client, cls.services)
+        cls.zone = get_zone(cls.api_client, cls.services)
+        cls.template = get_template(
+            cls.api_client,
+            cls.zone.id,
+            cls.services["ostype"]
+        )
+        cls.services["virtual_machine"]["zoneid"] = cls.zone.id
+        cls.services["virtual_machine"]["template"] = cls.template.id
+        cls.services["mode"] = cls.zone.networktype
+
+        cls.service_offering = ServiceOffering.create(
+            cls.api_client,
+            cls.services["service_offering"]
+        )
+        cls.vpc_off = VpcOffering.create(
+            cls.api_client,
+            cls.services["vpc_offering"]
+        )
+        cls.vpc_off.update(cls.api_client, state='Enabled')
+        hosts = Host.list(
+            cls.api_client,
+            zoneid=cls.zone.id,
+            listall=True,
+            type='Routing'
+        )
+
+        if isinstance(hosts, list):
+            for host in hosts:
+                Host.enableMaintenance(
+                    cls.api_client,
+                    id=host.id
+                )
+
+                timeout = cls.services["timeout"]
+                while True:
+                    time.sleep(cls.services["sleep"])
+                    hosts_states = Host.list(
+                        cls.api_client,
+                        id=host.id,
+                        listall=True
+                    )
+                    if hosts_states[0].resourcestate == 'PrepareForMaintenance':
+                        # Wait for sometimetill host goes in maintenance state
+                        time.sleep(cls.services["sleep"])
+                    elif hosts_states[0].resourcestate == 'Maintenance':
+                        time.sleep(cls.services["sleep"])
+                        break
+                    elif timeout == 0:
+                        raise unittest.SkipTest(
+                            "Failed to enable maintenance mode on %s" % host.name)
+                    timeout = timeout - 1
+
+        cls._cleanup = [
+            cls.service_offering,
+            cls.vpc_off
+        ]
+        return
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            #Cleanup resources used
+            cleanup_resources(cls.api_client, cls._cleanup)
+            hosts = Host.list(
+                cls.api_client,
+                zoneid=cls.zone.id,
+                listall=True,
+                type='Routing'
+            )
+            if isinstance(hosts, list):
+                for host in hosts:
+                    Host.cancelMaintenance(
+                        cls.api_client,
+                        id=host.id
+                    )
+                hosts_states = Host.list(
+                    cls.api_client,
+                    id=host.id,
+                    listall=True
+                )
+                if hosts_states[0].resourcestate != 'Enabled':
+                    raise Exception(
+                        "Failed to cancel maintenance mode on %s" % (host.name))
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self.account = Account.create(
+            self.apiclient,
+            self.services["account"],
+            admin=True,
+            domainid=self.domain.id
+        )
+        self.cleanup = [self.account]
+        return
+
+    def tearDown(self):
+        try:
+            #Clean up, terminate the created network offerings
+            cleanup_resources(self.apiclient, self.cleanup)
+            interval = list_configurations(
+                self.apiclient,
+                name='network.gc.interval'
+            )
+            wait = list_configurations(
+                self.apiclient,
+                name='network.gc.wait'
+            )
+            # Sleep to ensure that all resources are deleted
+            time.sleep(int(interval[0].value) + int(wait[0].value))
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def validate_vpc_offering(self, vpc_offering):
+        """Validates the VPC offering"""
+
+        self.debug("Check if the VPC offering is created successfully?")
+        vpc_offs = VpcOffering.list(
+            self.apiclient,
+            id=vpc_offering.id
+        )
+        self.assertEqual(
+            isinstance(vpc_offs, list),
+            True,
+            "List VPC offerings should return a valid list"
+        )
+        self.assertEqual(
+            vpc_offering.name,
+            vpc_offs[0].name,
+            "Name of the VPC offering should match with listVPCOff data"
+        )
+        self.debug(
+            "VPC offering is created successfully - %s" %
+            vpc_offering.name)
+        return
+
+    def validate_vpc_network(self, network, state=None):
+        """Validates the VPC network"""
+
+        self.debug("Check if the VPC network is created successfully?")
+        vpc_networks = VPC.list(
+            self.apiclient,
+            id=network.id
+        )
+        self.assertEqual(
+            isinstance(vpc_networks, list),
+            True,
+            "List VPC network should return a valid list"
+        )
+        self.assertEqual(
+            network.name,
+            vpc_networks[0].name,
+            "Name of the VPC network should match with listVPC data"
+        )
+        if state:
+            self.assertEqual(
+                vpc_networks[0].state,
+                state,
+                "VPC state should be '%s'" % state
+            )
+        self.debug("VPC network validated - %s" % network.name)
+        return
+
+    @attr(tags=["advanced", "intervlan"])
+    def test_01_create_vpc_host_maintenance(self):
+        """ Test VPC when host is in maintenance mode
+        """
+
+        # Validate the following
+        # 1. Put the host in maintenance mode.
+        # 2. Attempt to Create a VPC with cidr - 10.1.1.1/16
+        # 3. VPC will be created but will be in "Disabled" state
+
+        self.debug("creating a VPC network in the account: %s" %
+                   self.account.name)
+        self.services["vpc"]["cidr"] = '10.1.1.1/16'
+        vpc = VPC.create(
+            self.apiclient,
+            self.services["vpc"],
+            vpcofferingid=self.vpc_off.id,
+            zoneid=self.zone.id,
+            account=self.account.name,
+            domainid=self.account.domainid
+        )
+        self.validate_vpc_network(vpc, state='Disabled')
+        return
+
+    @attr(tags=["advanced", "intervlan"])
+    def test_02_create_vpc_wait_gc(self):
+        """ Test VPC when host is in maintenance mode and wait till nw gc
+        """
+
+        # Validate the following
+        # 1. Put the host in maintenance mode.
+        # 2. Attempt to Create a VPC with cidr - 10.1.1.1/16
+        # 3. Wait for the VPC GC thread to run.
+        # 3. VPC will be created but will be in "Disabled" state and should
+        #    get deleted
+
+        self.debug("creating a VPC network in the account: %s" %
+                   self.account.name)
+        self.services["vpc"]["cidr"] = '10.1.1.1/16'
+        vpc = VPC.create(
+            self.apiclient,
+            self.services["vpc"],
+            vpcofferingid=self.vpc_off.id,
+            zoneid=self.zone.id,
+            account=self.account.name,
+            domainid=self.account.domainid
+        )
+        self.validate_vpc_network(vpc, state='Disabled')
+        interval = list_configurations(
+            self.apiclient,
+            name='network.gc.interval'
+        )
+        wait = list_configurations(
+            self.apiclient,
+            name='network.gc.wait'
+        )
+        self.debug("Sleep till network gc thread runs..")
+        # Sleep to ensure that all resources are deleted
+        time.sleep(int(interval[0].value) + int(wait[0].value))
+        vpcs = VPC.list(
+            self.apiclient,
+            id=vpc.id,
+            listall=True
+        )
+        self.assertEqual(
+            vpcs,
+            None,
+            "List VPC should not return anything after network gc"
+        )
+        return

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/0587d3a4/test/integration/component/test_high_availability.py
----------------------------------------------------------------------
diff --git a/test/integration/component/test_high_availability.py b/test/integration/component/test_high_availability.py
deleted file mode 100644
index 7b0f78e..0000000
--- a/test/integration/component/test_high_availability.py
+++ /dev/null
@@ -1,1079 +0,0 @@
-#!/usr/bin/env python
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-""" P1 tests for high availability
-"""
-#Import Local Modules
-import marvin
-from nose.plugins.attrib import attr
-from marvin.cloudstackTestCase import *
-from marvin.cloudstackAPI import *
-from marvin.integration.lib.utils import *
-from marvin.integration.lib.base import *
-from marvin.integration.lib.common import *
-from marvin import remoteSSHClient
-import datetime
-
-
-class Services:
-    """Test network offering Services
-    """
-
-    def __init__(self):
-        self.services = {
-                         "account": {
-                                    "email": "test@test.com",
-                                    "firstname": "HA",
-                                    "lastname": "HA",
-                                    "username": "HA",
-                                    # Random characters are appended for unique
-                                    # username
-                                    "password": "password",
-                                    },
-                         "service_offering": {
-                                    "name": "Tiny Instance",
-                                    "displaytext": "Tiny Instance",
-                                    "cpunumber": 1,
-                                    "cpuspeed": 100,    # in MHz
-                                    "memory": 128,       # In MBs
-                                    },
-                         "lbrule": {
-                                    "name": "SSH",
-                                    "alg": "roundrobin",
-                                    # Algorithm used for load balancing
-                                    "privateport": 22,
-                                    "publicport": 2222,
-                                },
-                         "natrule": {
-                                    "privateport": 22,
-                                    "publicport": 22,
-                                    "protocol": "TCP"
-                                },
-                         "fw_rule": {
-                                    "startport": 1,
-                                    "endport": 6000,
-                                    "cidr": '55.55.0.0/11',
-                                    # Any network (For creating FW rule)
-                                },
-                         "virtual_machine": {
-                                    "displayname": "VM",
-                                    "username": "root",
-                                    "password": "password",
-                                    "ssh_port": 22,
-                                    "hypervisor": 'XenServer',
-                                    # Hypervisor type should be same as
-                                    # hypervisor type of cluster
-                                    "privateport": 22,
-                                    "publicport": 22,
-                                    "protocol": 'TCP',
-                                },
-                         "templates": {
-                                "displaytext": "Public Template",
-                                "name": "Public template",
-                                "ostype": 'CentOS 5.3 (64-bit)',
-                                "url": "http://download.cloud.com/releases/2.0.0/UbuntuServer-10-04-64bit.vhd.bz2",
-                                "hypervisor": 'XenServer',
-                                "format": 'VHD',
-                                "isfeatured": True,
-                                "ispublic": True,
-                                "isextractable": True,
-                                "templatefilter": 'self',
-                         },
-                         "ostype": 'CentOS 5.3 (64-bit)',
-                         # Cent OS 5.3 (64 bit)
-                         "sleep": 60,
-                         "timeout": 100,
-                         "mode": 'advanced'
-                    }
-
-
-class TestHighAvailability(cloudstackTestCase):
-
-    @classmethod
-    def setUpClass(cls):
-
-        cls.api_client = super(
-                               TestHighAvailability,
-                               cls
-                               ).getClsTestClient().getApiClient()
-        cls.services = Services().services
-        # Get Zone, Domain and templates
-        cls.domain = get_domain(
-                                cls.api_client,
-                                cls.services
-                            )
-        cls.zone = get_zone(
-                            cls.api_client,
-                            cls.services
-                            )
-        cls.pod = get_pod(
-                          cls.api_client,
-                          zoneid=cls.zone.id,
-                          services=cls.services
-                          )
-        cls.template = get_template(
-                                    cls.api_client,
-                                    cls.zone.id,
-                                    cls.services["ostype"]
-                            )
-        cls.services["virtual_machine"]["zoneid"] = cls.zone.id
-        cls.services["virtual_machine"]["template"] = cls.template.id
-
-        cls.service_offering = ServiceOffering.create(
-                                            cls.api_client,
-                                            cls.services["service_offering"],
-                                            offerha=True
-                                            )
-        cls._cleanup = [
-                        cls.service_offering,
-                        ]
-        return
-
-    @classmethod
-    def tearDownClass(cls):
-        try:
-            #Cleanup resources used
-            cleanup_resources(cls.api_client, cls._cleanup)
-        except Exception as e:
-            raise Exception("Warning: Exception during cleanup : %s" % e)
-        return
-
-    def setUp(self):
-        self.apiclient = self.testClient.getApiClient()
-        self.dbclient = self.testClient.getDbConnection()
-        self.account = Account.create(
-                                     self.apiclient,
-                                     self.services["account"],
-                                     admin=True,
-                                     domainid=self.domain.id
-                                     )
-        self.cleanup = [self.account]
-        return
-
-    def tearDown(self):
-        try:
-            #Clean up, terminate the created accounts, domains etc
-            cleanup_resources(self.apiclient, self.cleanup)
-        except Exception as e:
-            raise Exception("Warning: Exception during cleanup : %s" % e)
-        return
-
-    @attr(tags = ["advanced", "advancedns", "multihost"])
-    def test_01_host_maintenance_mode(self):
-        """Test host maintenance mode
-        """
-
-
-        # Validate the following
-        # 1. Create Vms. Acquire IP. Create port forwarding & load balancing
-        #    rules for Vms.
-        # 2. Host 1: put to maintenance mode. All Vms should failover to Host
-        #    2 in cluster. Vms should be in running state. All port forwarding
-        #    rules and load balancing Rules should work.
-        # 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
-        #    2 should succeed.
-        # 4. Host 1: cancel maintenance mode.
-        # 5. Host 2 : put to maintenance mode. All Vms should failover to
-        #    Host 1 in cluster.
-        # 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
-        #    host 1 should succeed.
-
-        hosts = Host.list(
-                          self.apiclient,
-                          zoneid=self.zone.id,
-                          resourcestate='Enabled',
-                          type='Routing'
-                          )
-        self.assertEqual(
-                         isinstance(hosts, list),
-                         True,
-                         "List hosts should return valid host response"
-                         )
-        self.assertGreaterEqual(
-                         len(hosts),
-                         2,
-                         "There must be two hosts present in a cluster"
-                        )
-        self.debug("Checking HA with hosts: %s, %s" % (
-                                                       hosts[0].name,
-                                                       hosts[1].name
-                                                       ))
-        self.debug("Deploying VM in account: %s" % self.account.name)
-        # Spawn an instance in that network
-        virtual_machine = VirtualMachine.create(
-                                  self.apiclient,
-                                  self.services["virtual_machine"],
-                                  accountid=self.account.name,
-                                  domainid=self.account.domainid,
-                                  serviceofferingid=self.service_offering.id
-                                  )
-        vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  id=virtual_machine.id,
-                                  listall=True
-                                  )
-        self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        vm = vms[0]
-        self.debug("Deployed VM on host: %s" % vm.hostid)
-        self.assertEqual(
-                         vm.state,
-                         "Running",
-                         "Deployed VM should be in RUnning state"
-                         )
-        networks = Network.list(
-                                self.apiclient,
-                                account=self.account.name,
-                                domainid=self.account.domainid,
-                                listall=True
-                                )
-        self.assertEqual(
-                    isinstance(networks, list),
-                    True,
-                    "List networks should return valid list for the account"
-                    )
-        network = networks[0]
-
-        self.debug("Associating public IP for account: %s" %
-                                            self.account.name)
-        public_ip = PublicIPAddress.create(
-                                    self.apiclient,
-                                    accountid=self.account.name,
-                                    zoneid=self.zone.id,
-                                    domainid=self.account.domainid,
-                                    networkid=network.id
-                                    )
-
-        self.debug("Associated %s with network %s" % (
-                                        public_ip.ipaddress.ipaddress,
-                                        network.id
-                                        ))
-        self.debug("Creating PF rule for IP address: %s" %
-                                        public_ip.ipaddress.ipaddress)
-        nat_rule = NATRule.create(
-                                 self.apiclient,
-                                 virtual_machine,
-                                 self.services["natrule"],
-                                 ipaddressid=public_ip.ipaddress.id
-                                 )
-
-        self.debug("Creating LB rule on IP with NAT: %s" %
-                                    public_ip.ipaddress.ipaddress)
-
-        # Create Load Balancer rule on IP already having NAT rule
-        lb_rule = LoadBalancerRule.create(
-                                    self.apiclient,
-                                    self.services["lbrule"],
-                                    ipaddressid=public_ip.ipaddress.id,
-                                    accountid=self.account.name
-                                    )
-        self.debug("Created LB rule with ID: %s" % lb_rule.id)
-
-        # Should be able to SSH VM
-        try:
-            self.debug("SSH into VM: %s" % virtual_machine.id)
-            ssh = virtual_machine.get_ssh_client(
-                                    ipaddress=public_ip.ipaddress.ipaddress)
-        except Exception as e:
-            self.fail("SSH Access failed for %s: %s" % \
-                      (virtual_machine.ipaddress, e)
-                      )
-
-        first_host = vm.hostid
-        self.debug("Enabling maintenance mode for host %s" % vm.hostid)
-        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
-        cmd.id = first_host
-        self.apiclient.prepareHostForMaintenance(cmd)
-
-        self.debug("Waiting for SSVMs to come up")
-        wait_for_ssvms(
-                       self.apiclient,
-                       zoneid=self.zone.id,
-                       podid=self.pod.id,
-                      )
-
-        timeout = self.services["timeout"]
-        # Poll and check state of VM while it migrates from one host to another
-        while True:
-            vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  id=virtual_machine.id,
-                                  listall=True
-                                  )
-            self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            vm = vms[0]
-
-            self.debug("VM 1 state: %s" % vm.state)
-            if vm.state in ["Stopping",
-                            "Stopped",
-                            "Running",
-                            "Starting",
-                            "Migrating"]:
-                if vm.state == "Running":
-                    break
-                else:
-                    time.sleep(self.services["sleep"])
-                    timeout = timeout - 1
-            else:
-                self.fail(
-                    "VM migration from one-host-to-other failed while enabling maintenance"
-                    )
-        second_host = vm.hostid
-        self.assertEqual(
-                vm.state,
-                "Running",
-                "VM should be in Running state after enabling host maintenance"
-                )
-        # Should be able to SSH VM
-        try:
-            self.debug("SSH into VM: %s" % virtual_machine.id)
-            ssh = virtual_machine.get_ssh_client(
-                                    ipaddress=public_ip.ipaddress.ipaddress)
-        except Exception as e:
-            self.fail("SSH Access failed for %s: %s" % \
-                      (virtual_machine.ipaddress, e)
-                      )
-        self.debug("Deploying VM in account: %s" % self.account.name)
-        # Spawn an instance on other host
-        virtual_machine_2 = VirtualMachine.create(
-                                  self.apiclient,
-                                  self.services["virtual_machine"],
-                                  accountid=self.account.name,
-                                  domainid=self.account.domainid,
-                                  serviceofferingid=self.service_offering.id
-                                  )
-        vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  id=virtual_machine_2.id,
-                                  listall=True
-                                  )
-        self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        vm = vms[0]
-        self.debug("Deployed VM on host: %s" % vm.hostid)
-        self.debug("VM 2 state: %s" % vm.state)
-        self.assertEqual(
-                         vm.state,
-                         "Running",
-                         "Deployed VM should be in Running state"
-                         )
-
-        self.debug("Canceling host maintenance for ID: %s" % first_host)
-        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
-        cmd.id = first_host
-        self.apiclient.cancelHostMaintenance(cmd)
-        self.debug("Maintenance mode canceled for host: %s" % first_host)
-
-        self.debug("Enabling maintenance mode for host %s" % second_host)
-        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
-        cmd.id = second_host
-        self.apiclient.prepareHostForMaintenance(cmd)
-        self.debug("Maintenance mode enabled for host: %s" % second_host)
-
-        self.debug("Waiting for SSVMs to come up")
-        wait_for_ssvms(
-                       self.apiclient,
-                       zoneid=self.zone.id,
-                       podid=self.pod.id,
-                      )
-
-        # Poll and check the status of VMs
-        timeout = self.services["timeout"]
-        while True:
-            vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  account=self.account.name,
-                                  domainid=self.account.domainid,
-                                  listall=True
-                                  )
-            self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            vm = vms[0]
-            self.debug(
-                "VM state after enabling maintenance on first host: %s" %
-                                                                    vm.state)
-            if vm.state in [
-                            "Stopping",
-                            "Stopped",
-                            "Running",
-                            "Starting",
-                            "Migrating"
-                            ]:
-                if vm.state == "Running":
-                    break
-                else:
-                    time.sleep(self.services["sleep"])
-                    timeout = timeout - 1
-            else:
-                self.fail(
-                    "VM migration from one-host-to-other failed while enabling maintenance"
-                    )
-
-                # Poll and check the status of VMs
-        timeout = self.services["timeout"]
-        while True:
-            vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  account=self.account.name,
-                                  domainid=self.account.domainid,
-                                  listall=True
-                                  )
-            self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            vm = vms[1]
-            self.debug(
-                "VM state after enabling maintenance on first host: %s" %
-                                                                    vm.state)
-            if vm.state in [
-                            "Stopping",
-                            "Stopped",
-                            "Running",
-                            "Starting",
-                            "Migrating"
-                            ]:
-                if vm.state == "Running":
-                    break
-                else:
-                    time.sleep(self.services["sleep"])
-                    timeout = timeout - 1
-            else:
-                self.fail(
-                    "VM migration from one-host-to-other failed while enabling maintenance"
-                    )
-
-        for vm in vms:
-            self.debug(
-                "VM states after enabling maintenance mode on host: %s - %s" %
-                                                    (first_host, vm.state))
-            self.assertEqual(
-                         vm.state,
-                         "Running",
-                         "Deployed VM should be in Running state"
-                         )
-
-        # Spawn an instance on other host
-        virtual_machine_3 = VirtualMachine.create(
-                                  self.apiclient,
-                                  self.services["virtual_machine"],
-                                  accountid=self.account.name,
-                                  domainid=self.account.domainid,
-                                  serviceofferingid=self.service_offering.id
-                                  )
-        vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  id=virtual_machine_3.id,
-                                  listall=True
-                                  )
-        self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        vm = vms[0]
-
-        self.debug("Deployed VM on host: %s" % vm.hostid)
-        self.debug("VM 3 state: %s" % vm.state)
-        self.assertEqual(
-                         vm.state,
-                         "Running",
-                         "Deployed VM should be in Running state"
-                         )
-
-        # Should be able to SSH VM
-        try:
-            self.debug("SSH into VM: %s" % virtual_machine.id)
-            ssh = virtual_machine.get_ssh_client(
-                                    ipaddress=public_ip.ipaddress.ipaddress)
-        except Exception as e:
-            self.fail("SSH Access failed for %s: %s" % \
-                      (virtual_machine.ipaddress, e)
-                      )
-
-        self.debug("Canceling host maintenance for ID: %s" % second_host)
-        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
-        cmd.id = second_host
-        self.apiclient.cancelHostMaintenance(cmd)
-        self.debug("Maintenance mode canceled for host: %s" % second_host)
-
-        self.debug("Waiting for SSVMs to come up")
-        wait_for_ssvms(
-                       self.apiclient,
-                       zoneid=self.zone.id,
-                       podid=self.pod.id,
-                      )
-        return
-
-    @attr(tags = ["advanced", "advancedns", "multihost"])
-    def test_02_host_maintenance_mode_with_activities(self):
-        """Test host maintenance mode with activities
-        """
-
-
-        # Validate the following
-        # 1. Create Vms. Acquire IP. Create port forwarding & load balancing
-        #    rules for Vms.
-        # 2. While activities are ongoing: Create snapshots, recurring
-        #    snapshots, create templates, download volumes, Host 1: put to
-        #    maintenance mode. All Vms should failover to Host 2 in cluster
-        #    Vms should be in running state. All port forwarding rules and
-        #    load balancing Rules should work.
-        # 3. After failover to Host 2 succeeds, deploy Vms. Deploy Vms on host
-        #    2 should succeed. All ongoing activities in step 3 should succeed
-        # 4. Host 1: cancel maintenance mode.
-        # 5. While activities are ongoing: Create snapshots, recurring
-        #    snapshots, create templates, download volumes, Host 2: put to
-        #    maintenance mode. All Vms should failover to Host 1 in cluster.
-        # 6. After failover to Host 1 succeeds, deploy VMs. Deploy Vms on
-        #    host 1 should succeed. All ongoing activities in step 6 should
-        #    succeed.
-
-        hosts = Host.list(
-                          self.apiclient,
-                          zoneid=self.zone.id,
-                          resourcestate='Enabled',
-                          type='Routing'
-                          )
-        self.assertEqual(
-                         isinstance(hosts, list),
-                         True,
-                         "List hosts should return valid host response"
-                         )
-        self.assertGreaterEqual(
-                         len(hosts),
-                         2,
-                         "There must be two hosts present in a cluster"
-                        )
-        self.debug("Checking HA with hosts: %s, %s" % (
-                                                       hosts[0].name,
-                                                       hosts[1].name
-                                                       ))
-        self.debug("Deploying VM in account: %s" % self.account.name)
-        # Spawn an instance in that network
-        virtual_machine = VirtualMachine.create(
-                                  self.apiclient,
-                                  self.services["virtual_machine"],
-                                  accountid=self.account.name,
-                                  domainid=self.account.domainid,
-                                  serviceofferingid=self.service_offering.id
-                                  )
-        vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  id=virtual_machine.id,
-                                  listall=True
-                                  )
-        self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        vm = vms[0]
-        self.debug("Deployed VM on host: %s" % vm.hostid)
-        self.assertEqual(
-                         vm.state,
-                         "Running",
-                         "Deployed VM should be in RUnning state"
-                         )
-        networks = Network.list(
-                                self.apiclient,
-                                account=self.account.name,
-                                domainid=self.account.domainid,
-                                listall=True
-                                )
-        self.assertEqual(
-                    isinstance(networks, list),
-                    True,
-                    "List networks should return valid list for the account"
-                    )
-        network = networks[0]
-
-        self.debug("Associating public IP for account: %s" %
-                                            self.account.name)
-        public_ip = PublicIPAddress.create(
-                                    self.apiclient,
-                                    accountid=self.account.name,
-                                    zoneid=self.zone.id,
-                                    domainid=self.account.domainid,
-                                    networkid=network.id
-                                    )
-
-        self.debug("Associated %s with network %s" % (
-                                        public_ip.ipaddress.ipaddress,
-                                        network.id
-                                        ))
-        self.debug("Creating PF rule for IP address: %s" %
-                                        public_ip.ipaddress.ipaddress)
-        nat_rule = NATRule.create(
-                                 self.apiclient,
-                                 virtual_machine,
-                                 self.services["natrule"],
-                                 ipaddressid=public_ip.ipaddress.id
-                                 )
-
-        self.debug("Creating LB rule on IP with NAT: %s" %
-                                    public_ip.ipaddress.ipaddress)
-
-        # Create Load Balancer rule on IP already having NAT rule
-        lb_rule = LoadBalancerRule.create(
-                                    self.apiclient,
-                                    self.services["lbrule"],
-                                    ipaddressid=public_ip.ipaddress.id,
-                                    accountid=self.account.name
-                                    )
-        self.debug("Created LB rule with ID: %s" % lb_rule.id)
-
-        # Should be able to SSH VM
-        try:
-            self.debug("SSH into VM: %s" % virtual_machine.id)
-            ssh = virtual_machine.get_ssh_client(
-                                    ipaddress=public_ip.ipaddress.ipaddress)
-        except Exception as e:
-            self.fail("SSH Access failed for %s: %s" % \
-                      (virtual_machine.ipaddress, e)
-                      )
-        # Get the Root disk of VM
-        volumes = list_volumes(
-                            self.apiclient,
-                            virtualmachineid=virtual_machine.id,
-                            type='ROOT',
-                            listall=True
-                            )
-        volume = volumes[0]
-        self.debug(
-            "Root volume of VM(%s): %s" % (
-                                            virtual_machine.name,
-                                            volume.name
-                                            ))
-        # Create a snapshot from the ROOTDISK
-        self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
-        snapshot = Snapshot.create(self.apiclient, volumes[0].id)
-        self.debug("Snapshot created: ID - %s" % snapshot.id)
-
-        snapshots = list_snapshots(
-                                   self.apiclient,
-                                   id=snapshot.id,
-                                   listall=True
-                                   )
-        self.assertEqual(
-                            isinstance(snapshots, list),
-                            True,
-                            "Check list response returns a valid list"
-                        )
-        self.assertNotEqual(
-                            snapshots,
-                            None,
-                            "Check if result exists in list snapshots call"
-                            )
-        self.assertEqual(
-                            snapshots[0].id,
-                            snapshot.id,
-                            "Check snapshot id in list resources call"
-                        )
-
-        # Generate template from the snapshot
-        self.debug("Generating template from snapshot: %s" % snapshot.name)
-        template = Template.create_from_snapshot(
-                                                 self.apiclient,
-                                                 snapshot,
-                                                 self.services["templates"]
-                                                 )
-        self.debug("Created template from snapshot: %s" % template.id)
-
-        templates = list_templates(
-                                self.apiclient,
-                                templatefilter=\
-                                self.services["templates"]["templatefilter"],
-                                id=template.id
-                                )
-
-        self.assertEqual(
-                isinstance(templates, list),
-                True,
-                "List template call should return the newly created template"
-                )
-
-        self.assertEqual(
-                    templates[0].isready,
-                    True,
-                    "The newly created template should be in ready state"
-                    )
-
-        first_host = vm.hostid
-        self.debug("Enabling maintenance mode for host %s" % vm.hostid)
-        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
-        cmd.id = first_host
-        self.apiclient.prepareHostForMaintenance(cmd)
-
-        self.debug("Waiting for SSVMs to come up")
-        wait_for_ssvms(
-                       self.apiclient,
-                       zoneid=self.zone.id,
-                       podid=self.pod.id,
-                      )
-
-        timeout = self.services["timeout"]
-        # Poll and check state of VM while it migrates from one host to another
-        while True:
-            vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  id=virtual_machine.id,
-                                  listall=True
-                                  )
-            self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            vm = vms[0]
-
-            self.debug("VM 1 state: %s" % vm.state)
-            if vm.state in ["Stopping",
-                            "Stopped",
-                            "Running",
-                            "Starting",
-                            "Migrating"]:
-                if vm.state == "Running":
-                    break
-                else:
-                    time.sleep(self.services["sleep"])
-                    timeout = timeout - 1
-            else:
-                self.fail(
-                    "VM migration from one-host-to-other failed while enabling maintenance"
-                    )
-        second_host = vm.hostid
-        self.assertEqual(
-                vm.state,
-                "Running",
-                "VM should be in Running state after enabling host maintenance"
-                )
-        # Should be able to SSH VM
-        try:
-            self.debug("SSH into VM: %s" % virtual_machine.id)
-            ssh = virtual_machine.get_ssh_client(
-                                    ipaddress=public_ip.ipaddress.ipaddress)
-        except Exception as e:
-            self.fail("SSH Access failed for %s: %s" % \
-                      (virtual_machine.ipaddress, e)
-                      )
-        self.debug("Deploying VM in account: %s" % self.account.name)
-        # Spawn an instance on other host
-        virtual_machine_2 = VirtualMachine.create(
-                                  self.apiclient,
-                                  self.services["virtual_machine"],
-                                  accountid=self.account.name,
-                                  domainid=self.account.domainid,
-                                  serviceofferingid=self.service_offering.id
-                                  )
-        vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  id=virtual_machine_2.id,
-                                  listall=True
-                                  )
-        self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        vm = vms[0]
-        self.debug("Deployed VM on host: %s" % vm.hostid)
-        self.debug("VM 2 state: %s" % vm.state)
-        self.assertEqual(
-                         vm.state,
-                         "Running",
-                         "Deployed VM should be in Running state"
-                         )
-
-        self.debug("Canceling host maintenance for ID: %s" % first_host)
-        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
-        cmd.id = first_host
-        self.apiclient.cancelHostMaintenance(cmd)
-        self.debug("Maintenance mode canceled for host: %s" % first_host)
-
-        # Get the Root disk of VM
-        volumes = list_volumes(
-                            self.apiclient,
-                            virtualmachineid=virtual_machine_2.id,
-                            type='ROOT',
-                            listall=True
-                            )
-        volume = volumes[0]
-        self.debug(
-            "Root volume of VM(%s): %s" % (
-                                            virtual_machine_2.name,
-                                            volume.name
-                                            ))
-        # Create a snapshot from the ROOTDISK
-        self.debug("Creating snapshot on ROOT volume: %s" % volume.name)
-        snapshot = Snapshot.create(self.apiclient, volumes[0].id)
-        self.debug("Snapshot created: ID - %s" % snapshot.id)
-
-        snapshots = list_snapshots(
-                                   self.apiclient,
-                                   id=snapshot.id,
-                                   listall=True
-                                   )
-        self.assertEqual(
-                            isinstance(snapshots, list),
-                            True,
-                            "Check list response returns a valid list"
-                        )
-        self.assertNotEqual(
-                            snapshots,
-                            None,
-                            "Check if result exists in list snapshots call"
-                            )
-        self.assertEqual(
-                            snapshots[0].id,
-                            snapshot.id,
-                            "Check snapshot id in list resources call"
-                        )
-
-        # Generate template from the snapshot
-        self.debug("Generating template from snapshot: %s" % snapshot.name)
-        template = Template.create_from_snapshot(
-                                                 self.apiclient,
-                                                 snapshot,
-                                                 self.services["templates"]
-                                                 )
-        self.debug("Created template from snapshot: %s" % template.id)
-
-        templates = list_templates(
-                                self.apiclient,
-                                templatefilter=\
-                                self.services["templates"]["templatefilter"],
-                                id=template.id
-                                )
-
-        self.assertEqual(
-                isinstance(templates, list),
-                True,
-                "List template call should return the newly created template"
-                )
-
-        self.assertEqual(
-                    templates[0].isready,
-                    True,
-                    "The newly created template should be in ready state"
-                    )
-
-        self.debug("Enabling maintenance mode for host %s" % second_host)
-        cmd = prepareHostForMaintenance.prepareHostForMaintenanceCmd()
-        cmd.id = second_host
-        self.apiclient.prepareHostForMaintenance(cmd)
-        self.debug("Maintenance mode enabled for host: %s" % second_host)
-
-        self.debug("Waiting for SSVMs to come up")
-        wait_for_ssvms(
-                       self.apiclient,
-                       zoneid=self.zone.id,
-                       podid=self.pod.id,
-                      )
-
-        # Poll and check the status of VMs
-        timeout = self.services["timeout"]
-        while True:
-            vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  account=self.account.name,
-                                  domainid=self.account.domainid,
-                                  listall=True
-                                  )
-            self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            vm = vms[0]
-            self.debug(
-                "VM state after enabling maintenance on first host: %s" %
-                                                                    vm.state)
-            if vm.state in ["Stopping",
-                            "Stopped",
-                            "Running",
-                            "Starting",
-                            "Migrating"]:
-                if vm.state == "Running":
-                    break
-                else:
-                    time.sleep(self.services["sleep"])
-                    timeout = timeout - 1
-            else:
-                self.fail(
-                    "VM migration from one-host-to-other failed while enabling maintenance"
-                    )
-
-        # Poll and check the status of VMs
-        timeout = self.services["timeout"]
-        while True:
-            vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  account=self.account.name,
-                                  domainid=self.account.domainid,
-                                  listall=True
-                                  )
-            self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-            vm = vms[1]
-            self.debug(
-                "VM state after enabling maintenance on first host: %s" %
-                                                                    vm.state)
-            if vm.state in ["Stopping",
-                            "Stopped",
-                            "Running",
-                            "Starting",
-                            "Migrating"]:
-                if vm.state == "Running":
-                    break
-                else:
-                    time.sleep(self.services["sleep"])
-                    timeout = timeout - 1
-            else:
-                self.fail(
-                    "VM migration from one-host-to-other failed while enabling maintenance"
-                    )
-
-        for vm in vms:
-            self.debug(
-                "VM states after enabling maintenance mode on host: %s - %s" %
-                                                    (first_host, vm.state))
-            self.assertEqual(
-                         vm.state,
-                         "Running",
-                         "Deployed VM should be in Running state"
-                         )
-
-        # Spawn an instance on other host
-        virtual_machine_3 = VirtualMachine.create(
-                                  self.apiclient,
-                                  self.services["virtual_machine"],
-                                  accountid=self.account.name,
-                                  domainid=self.account.domainid,
-                                  serviceofferingid=self.service_offering.id
-                                  )
-        vms = VirtualMachine.list(
-                                  self.apiclient,
-                                  id=virtual_machine_3.id,
-                                  listall=True
-                                  )
-        self.assertEqual(
-                    isinstance(vms, list),
-                    True,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        self.assertNotEqual(
-                    len(vms),
-                    0,
-                    "List VMs should return valid response for deployed VM"
-                    )
-        vm = vms[0]
-
-        self.debug("Deployed VM on host: %s" % vm.hostid)
-        self.debug("VM 3 state: %s" % vm.state)
-        self.assertEqual(
-                         vm.state,
-                         "Running",
-                         "Deployed VM should be in Running state"
-                         )
-
-        self.debug("Canceling host maintenance for ID: %s" % second_host)
-        cmd = cancelHostMaintenance.cancelHostMaintenanceCmd()
-        cmd.id = second_host
-        self.apiclient.cancelHostMaintenance(cmd)
-        self.debug("Maintenance mode canceled for host: %s" % second_host)
-
-        self.debug("Waiting for SSVMs to come up")
-        wait_for_ssvms(
-                       self.apiclient,
-                       zoneid=self.zone.id,
-                       podid=self.pod.id,
-                      )
-        return