You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by wi...@apache.org on 2015/07/27 10:42:58 UTC
[10/50] [abbrv] git commit: updated refs/heads/reporter to 3e1816d
CLOUDSTACK-8628: Send an alert when fencing a KVM host failed
Also change the logging a bit so that you get useful logs when
not running on DEBUG level
Signed-off-by: Wido den Hollander <wi...@widodh.nl>
This closes #580
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/fbe3b04a
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/fbe3b04a
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/fbe3b04a
Branch: refs/heads/reporter
Commit: fbe3b04a43ff7380a6874a2c331f372aa59ccf3d
Parents: d1f76a2
Author: Wido den Hollander <wi...@widodh.nl>
Authored: Thu Jul 16 16:15:20 2015 +0200
Committer: Wido den Hollander <wi...@widodh.nl>
Committed: Fri Jul 17 12:33:11 2015 +0200
----------------------------------------------------------------------
server/src/com/cloud/ha/KVMFencer.java | 26 +++++++++++++++---------
server/test/com/cloud/ha/KVMFencerTest.java | 22 ++++++++++++++++++++
2 files changed, 38 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/fbe3b04a/server/src/com/cloud/ha/KVMFencer.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/KVMFencer.java b/server/src/com/cloud/ha/KVMFencer.java
index 7392dff..b5834ef 100644
--- a/server/src/com/cloud/ha/KVMFencer.java
+++ b/server/src/com/cloud/ha/KVMFencer.java
@@ -26,6 +26,7 @@ import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
+import com.cloud.alert.AlertManager;
import com.cloud.agent.api.FenceAnswer;
import com.cloud.agent.api.FenceCommand;
import com.cloud.exception.AgentUnavailableException;
@@ -48,6 +49,8 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
@Inject
AgentManager _agentMgr;
@Inject
+ AlertManager _alertMgr;
+ @Inject
ResourceManager _resourceMgr;
@Override
@@ -75,18 +78,22 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
@Override
public Boolean fenceOff(VirtualMachine vm, Host host) {
if (host.getHypervisorType() != HypervisorType.KVM && host.getHypervisorType() != HypervisorType.LXC) {
- s_logger.debug("Don't know how to fence non kvm hosts " + host.getHypervisorType());
+ s_logger.warn("Don't know how to fence non kvm hosts " + host.getHypervisorType());
return null;
}
List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId());
FenceCommand fence = new FenceCommand(vm, host);
+ int i = 0;
for (HostVO h : hosts) {
if (h.getHypervisorType() == HypervisorType.KVM || h.getHypervisorType() == HypervisorType.LXC) {
if (h.getStatus() != Status.Up) {
continue;
}
+
+ i++;
+
if (h.getId() == host.getId()) {
continue;
}
@@ -94,14 +101,10 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
try {
answer = (FenceAnswer)_agentMgr.send(h.getId(), fence);
} catch (AgentUnavailableException e) {
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
- }
+ s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
continue;
} catch (OperationTimedoutException e) {
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
- }
+ s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
continue;
}
if (answer != null && answer.getResult()) {
@@ -110,9 +113,12 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
}
}
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Unable to fence off " + vm.toString() + " on " + host.toString());
- }
+ _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(),
+ "Unable to fence off host: " + host.getId(),
+ "Fencing off host " + host.getId() + " did not succeed after asking " + i + " hosts. " +
+ "Check Agent logs for more information.");
+
+ s_logger.error("Unable to fence off " + vm.toString() + " on " + host.toString());
return false;
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/fbe3b04a/server/test/com/cloud/ha/KVMFencerTest.java
----------------------------------------------------------------------
diff --git a/server/test/com/cloud/ha/KVMFencerTest.java b/server/test/com/cloud/ha/KVMFencerTest.java
index cdd13b6..da120af 100644
--- a/server/test/com/cloud/ha/KVMFencerTest.java
+++ b/server/test/com/cloud/ha/KVMFencerTest.java
@@ -31,6 +31,7 @@ import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import com.cloud.agent.AgentManager;
+import com.cloud.alert.AlertManager;
import com.cloud.agent.api.FenceAnswer;
import com.cloud.agent.api.FenceCommand;
import com.cloud.exception.AgentUnavailableException;
@@ -50,6 +51,8 @@ public class KVMFencerTest {
@Mock
AgentManager agentManager;
@Mock
+ AlertManager alertMgr;
+ @Mock
ResourceManager resourceManager;
KVMFencer fencer;
@@ -58,6 +61,7 @@ public class KVMFencerTest {
public void setup() {
fencer = new KVMFencer();
fencer._agentMgr = agentManager;
+ fencer._alertMgr = alertMgr;
fencer._hostDao = hostDao;
fencer._resourceMgr = resourceManager;
}
@@ -67,6 +71,8 @@ public class KVMFencerTest {
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
Mockito.when(host.getId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -80,6 +86,8 @@ public class KVMFencerTest {
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getStatus()).thenReturn(Status.Down);
Mockito.when(host.getId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -94,12 +102,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+ Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+ Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -118,12 +130,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+ Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+ Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -141,12 +157,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+ Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+ Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -165,6 +185,8 @@ public class KVMFencerTest {
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.Any);
Mockito.when(host.getStatus()).thenReturn(Status.Down);
Mockito.when(host.getId()).thenReturn(1l);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
Mockito.when(resourceManager.listAllHostsInCluster(1l)).thenReturn(Collections.singletonList(host));