[vcl] 02/04: libvirt.pm: modified power_off: added check for "SIGKILL: Device or resource busy" in output of virsh destroy command - we've experienced times when a domain was considered busy but could be destroyed a few seconds later, this waits 30 seconds and tries again

[jf...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jfthomps pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/vcl.git

commit 655566bd1fb0c9996e17e25fc2b6fa4c2bd164d5
Author: Josh Thompson <jf...@ncsu.edu>
AuthorDate: Thu Jul 11 11:51:10 2019 -0400

    libvirt.pm: modified power_off: added check for "SIGKILL: Device or resource busy" in output of virsh destroy command - we've experienced times when a domain was considered busy but could be destroyed a few seconds later, this waits 30 seconds and tries again
---
 .../lib/VCL/Module/Provisioning/libvirt.pm         | 38 ++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/managementnode/lib/VCL/Module/Provisioning/libvirt.pm b/managementnode/lib/VCL/Module/Provisioning/libvirt.pm
index bee452d..04cccc6 100644
--- a/managementnode/lib/VCL/Module/Provisioning/libvirt.pm
+++ b/managementnode/lib/VCL/Module/Provisioning/libvirt.pm
@@ -779,6 +779,44 @@ sub power_off {
 		notify($ERRORS{'DEBUG'}, 0, "'$domain_name' domain is not running on $node_name");
 		return 1;
 	}
+	elsif (grep(/SIGKILL: Device or resource busy/i, @$output)) {
+		notify($ERRORS{'DEBUG'}, 0, "Failed to stop '$domain_name' domain; \"SIGKILL: Device or resource busy\" encountered, waiting 30 seconds and trying again");
+		# libvirt could not kill VM, wait and try again
+		sleep 30;
+		my $command2 = "virsh -q list --all";
+		($exit_status, $output) = $self->vmhost_os->execute($command2);
+		my ($id, $name, $state);
+		for my $line (@$output) {
+			($id, $name, $state) = $line =~ /^\s*([\d\-]+)\s+(.+?)\s+(\w+|shut off|in shutdown)$/g;
+			next if (!defined($id));
+			last if ($name eq $domain_name);
+		}
+		if ($name ne $domain_name) {
+			notify($ERRORS{'WARNING'}, 0, "2nd try: $domain_name not found when running 'virsh list' on $node_name");
+			return;
+		}
+		if ($id eq '-') {
+			notify($ERRORS{'DEBUG'}, 0, "'$domain_name' domain now stopped");
+			return 1;
+		}
+		($exit_status, $output) = $self->vmhost_os->execute($command);
+		if (!defined($output)) {
+			notify($ERRORS{'WARNING'}, 0, "2nd try: failed to execute virsh command to destroy '$domain_name' domain on $node_name");
+			return;
+		}
+		elsif ($exit_status eq '0') {
+			notify($ERRORS{'OK'}, 0, "2nd try: destroyed '$domain_name' domain on $node_name");
+			return 1;
+		}
+		elsif (grep(/domain is not running/i, @$output)) {
+			notify($ERRORS{'DEBUG'}, 0, "2nd try: '$domain_name' domain is not running on $node_name");
+			return 1;
+		}
+		else {
+			notify($ERRORS{'WARNING'}, 0, "2nd try: failed to destroy '$domain_name' domain on $node_name\ncommand: $command\nexit status: $exit_status\noutput:\n" . join("\n", @$output));
+			return;
+		}
+	}
 	else {
 		notify($ERRORS{'WARNING'}, 0, "failed to destroy '$domain_name' domain on $node_name\ncommand: $command\nexit status: $exit_status\noutput:\n" . join("\n", @$output));
 		return;