You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ro...@apache.org on 2019/03/26 10:03:49 UTC
[cloudstack] branch 4.11 updated: tests: fix some Marvin smoketests
(#2869)
This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch 4.11
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.11 by this push:
new 8b25fdf tests: fix some Marvin smoketests (#2869)
8b25fdf is described below
commit 8b25fdf520abdd142d125e82dd32335a36a133cd
Author: Paul Angus <pa...@shapeblue.com>
AuthorDate: Tue Mar 26 10:03:40 2019 +0000
tests: fix some Marvin smoketests (#2869)
Fixes intermittent failures:
- Add a pause to avoid tests restarting before VRs recovering from HA have fully booted.
- Add some pauses to allow services to restart and hosts to reconnect before continuing tests.
- Adding a loop around this method because sometimes VRs are overloaded and just respond with exception, so it'll catch it and try 5 times with 30sec cooldown. If that fails as well it'll fail the test.
- wait until host is up using explicit check
---
.../smoke/test_deploy_virtio_scsi_vm.py | 1 +
test/integration/smoke/test_hostha_kvm.py | 5 ++-
test/integration/smoke/test_vm_life_cycle.py | 25 +++++++++++-
tools/marvin/marvin/lib/base.py | 44 ++++++++++++++--------
4 files changed, 56 insertions(+), 19 deletions(-)
diff --git a/test/integration/smoke/test_deploy_virtio_scsi_vm.py b/test/integration/smoke/test_deploy_virtio_scsi_vm.py
index df54c43..8969097 100644
--- a/test/integration/smoke/test_deploy_virtio_scsi_vm.py
+++ b/test/integration/smoke/test_deploy_virtio_scsi_vm.py
@@ -96,6 +96,7 @@ class TestDeployVirtioSCSIVM(cloudstackTestCase):
cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
cls.pod = get_pod(cls.apiclient, cls.zone.id)
cls.services['mode'] = cls.zone.networktype
+ cls.cleanup = []
if cls.hypervisor.lower() not in ['kvm']:
cls.hypervisorNotSupported = True
return
diff --git a/test/integration/smoke/test_hostha_kvm.py b/test/integration/smoke/test_hostha_kvm.py
index a4de07e..cd4a2d4 100644
--- a/test/integration/smoke/test_hostha_kvm.py
+++ b/test/integration/smoke/test_hostha_kvm.py
@@ -274,9 +274,12 @@ class TestHAKVM(cloudstackTestCase):
Tests Enable HA when host is in Maintenance mode, should be Ineligible
"""
self.logger.debug("Starting test_hostha_enable_ha_when_host_in_maintenance")
-
+ self.logger.debug("Pausing to wait for VMs to have finished starting")
+ time.sleep(300)
+
# Enable HA
self.configureAndEnableHostHa()
+
# Prepare for maintenance Host
self.setHostToMaintanance(self.host.id)
diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py
index 5906a94..d1962c4 100644
--- a/test/integration/smoke/test_vm_life_cycle.py
+++ b/test/integration/smoke/test_vm_life_cycle.py
@@ -900,6 +900,23 @@ class TestSecuredVmMigration(cloudstackTestCase):
vm_response = VirtualMachine.list(self.apiclient, id=vm.id)[0]
self.assertEqual(vm_response.hostid, dest_host.id, "Check destination host ID of migrated VM")
+ def waitUntilHostInState(self, hostId, state="Up", interval=5, retries=20):
+ while retries > -1:
+ print("Waiting for host: %s to be %s. %s retries left." % (hostId, state, retries))
+ time.sleep(interval)
+ host = Host.list(
+ self.apiclient,
+ hostid=hostId,
+ type='Routing'
+ )[0]
+ if host.state != state:
+ if retries >= 0:
+ retries = retries - 1
+ continue
+ else:
+ print("Host %s now showing as %s" % (hostId, state))
+ return
+
def unsecure_host(self, host):
SshClient(host.ipaddress, port=22, user=self.hostConfig["username"], passwd=self.hostConfig["password"])\
.execute("rm -f /etc/cloudstack/agent/cloud* && \
@@ -907,9 +924,10 @@ class TestSecuredVmMigration(cloudstackTestCase):
sed -i 's/listen_tcp.*/listen_tcp=1/g' /etc/libvirt/libvirtd.conf && \
sed -i '/.*_file=.*/d' /etc/libvirt/libvirtd.conf && \
service libvirtd restart && \
+ sleep 30 && \
service cloudstack-agent restart")
-
- time.sleep(10)
+ print("Unsecuring Host: %s" % (host.name))
+ self.waitUntilHostInState(hostId=host.id, state="Up")
self.check_connection(host=host, secured='false')
return host
@@ -921,6 +939,8 @@ class TestSecuredVmMigration(cloudstackTestCase):
self.apiclient.provisionCertificate(cmd)
for host in self.hosts:
+ print("Securing Host %s" % host.name)
+ self.waitUntilHostInState(hostId=host.id, state="Up")
self.check_connection(secured='true', host=host)
def deploy_vm(self, origin_host):
@@ -971,6 +991,7 @@ class TestSecuredVmMigration(cloudstackTestCase):
vm = self.deploy_vm(src_host)
self.cleanup.append(vm)
+ self.debug("Securing Host(s)")
dest_host = self.get_target_host(secured='true', virtualmachineid=vm.id)
self.migrate_and_check(vm, src_host, dest_host)
diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py
index e8166fb..ede8256 100755
--- a/tools/marvin/marvin/lib/base.py
+++ b/tools/marvin/marvin/lib/base.py
@@ -605,22 +605,34 @@ class VirtualMachine:
return VirtualMachine(virtual_machine.__dict__, services)
# program ssh access over NAT via PF
- if mode.lower() == 'advanced':
- cls.access_ssh_over_nat(
- apiclient,
- services,
- virtual_machine,
- allow_egress=allow_egress,
- networkid=cmd.networkids[0] if cmd.networkids else None)
- elif mode.lower() == 'basic':
- if virtual_machine.publicip is not None:
- # EIP/ELB (netscaler) enabled zone
- vm_ssh_ip = virtual_machine.publicip
- else:
- # regular basic zone with security group
- vm_ssh_ip = virtual_machine.nic[0].ipaddress
- virtual_machine.ssh_ip = vm_ssh_ip
- virtual_machine.public_ip = vm_ssh_ip
+ retries = 5
+ interval = 30
+ while retries > 0:
+ time.sleep(interval)
+ try:
+ if mode.lower() == 'advanced':
+ cls.access_ssh_over_nat(
+ apiclient,
+ services,
+ virtual_machine,
+ allow_egress=allow_egress,
+ networkid=cmd.networkids[0] if cmd.networkids else None)
+ elif mode.lower() == 'basic':
+ if virtual_machine.publicip is not None:
+ # EIP/ELB (netscaler) enabled zone
+ vm_ssh_ip = virtual_machine.publicip
+ else:
+ # regular basic zone with security group
+ vm_ssh_ip = virtual_machine.nic[0].ipaddress
+ virtual_machine.ssh_ip = vm_ssh_ip
+ virtual_machine.public_ip = vm_ssh_ip
+ break
+ except Exception as e:
+ if retries >= 0:
+ retries = retries - 1
+ continue
+ raise Exception(
+ "The following exception appeared while programming ssh access - %s" % e)
return VirtualMachine(virtual_machine.__dict__, services)