You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by wi...@apache.org on 2015/07/17 12:33:21 UTC
[1/2] git commit: updated refs/heads/master to fbe3b04
Repository: cloudstack
Updated Branches:
refs/heads/master 8d96aab9b -> fbe3b04a4
CLOUDSTACK-8628: Send an alert when fencing a KVM host failed
Also change the logging a bit so that you get useful logs when
not running on DEBUG level
Signed-off-by: Wido den Hollander <wi...@widodh.nl>
This closes #580
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/fbe3b04a
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/fbe3b04a
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/fbe3b04a
Branch: refs/heads/master
Commit: fbe3b04a43ff7380a6874a2c331f372aa59ccf3d
Parents: d1f76a2
Author: Wido den Hollander <wi...@widodh.nl>
Authored: Thu Jul 16 16:15:20 2015 +0200
Committer: Wido den Hollander <wi...@widodh.nl>
Committed: Fri Jul 17 12:33:11 2015 +0200
----------------------------------------------------------------------
server/src/com/cloud/ha/KVMFencer.java | 26 +++++++++++++++---------
server/test/com/cloud/ha/KVMFencerTest.java | 22 ++++++++++++++++++++
2 files changed, 38 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/fbe3b04a/server/src/com/cloud/ha/KVMFencer.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/KVMFencer.java b/server/src/com/cloud/ha/KVMFencer.java
index 7392dff..b5834ef 100644
--- a/server/src/com/cloud/ha/KVMFencer.java
+++ b/server/src/com/cloud/ha/KVMFencer.java
@@ -26,6 +26,7 @@ import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
+import com.cloud.alert.AlertManager;
import com.cloud.agent.api.FenceAnswer;
import com.cloud.agent.api.FenceCommand;
import com.cloud.exception.AgentUnavailableException;
@@ -48,6 +49,8 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
@Inject
AgentManager _agentMgr;
@Inject
+ AlertManager _alertMgr;
+ @Inject
ResourceManager _resourceMgr;
@Override
@@ -75,18 +78,22 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
@Override
public Boolean fenceOff(VirtualMachine vm, Host host) {
if (host.getHypervisorType() != HypervisorType.KVM && host.getHypervisorType() != HypervisorType.LXC) {
- s_logger.debug("Don't know how to fence non kvm hosts " + host.getHypervisorType());
+ s_logger.warn("Don't know how to fence non kvm hosts " + host.getHypervisorType());
return null;
}
List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId());
FenceCommand fence = new FenceCommand(vm, host);
+ int i = 0;
for (HostVO h : hosts) {
if (h.getHypervisorType() == HypervisorType.KVM || h.getHypervisorType() == HypervisorType.LXC) {
if (h.getStatus() != Status.Up) {
continue;
}
+
+ i++;
+
if (h.getId() == host.getId()) {
continue;
}
@@ -94,14 +101,10 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
try {
answer = (FenceAnswer)_agentMgr.send(h.getId(), fence);
} catch (AgentUnavailableException e) {
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
- }
+ s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
continue;
} catch (OperationTimedoutException e) {
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
- }
+ s_logger.info("Moving on to the next host because " + h.toString() + " is unavailable");
continue;
}
if (answer != null && answer.getResult()) {
@@ -110,9 +113,12 @@ public class KVMFencer extends AdapterBase implements FenceBuilder {
}
}
- if (s_logger.isDebugEnabled()) {
- s_logger.debug("Unable to fence off " + vm.toString() + " on " + host.toString());
- }
+ _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(),
+ "Unable to fence off host: " + host.getId(),
+ "Fencing off host " + host.getId() + " did not succeed after asking " + i + " hosts. " +
+ "Check Agent logs for more information.");
+
+ s_logger.error("Unable to fence off " + vm.toString() + " on " + host.toString());
return false;
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/fbe3b04a/server/test/com/cloud/ha/KVMFencerTest.java
----------------------------------------------------------------------
diff --git a/server/test/com/cloud/ha/KVMFencerTest.java b/server/test/com/cloud/ha/KVMFencerTest.java
index cdd13b6..da120af 100644
--- a/server/test/com/cloud/ha/KVMFencerTest.java
+++ b/server/test/com/cloud/ha/KVMFencerTest.java
@@ -31,6 +31,7 @@ import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import com.cloud.agent.AgentManager;
+import com.cloud.alert.AlertManager;
import com.cloud.agent.api.FenceAnswer;
import com.cloud.agent.api.FenceCommand;
import com.cloud.exception.AgentUnavailableException;
@@ -50,6 +51,8 @@ public class KVMFencerTest {
@Mock
AgentManager agentManager;
@Mock
+ AlertManager alertMgr;
+ @Mock
ResourceManager resourceManager;
KVMFencer fencer;
@@ -58,6 +61,7 @@ public class KVMFencerTest {
public void setup() {
fencer = new KVMFencer();
fencer._agentMgr = agentManager;
+ fencer._alertMgr = alertMgr;
fencer._hostDao = hostDao;
fencer._resourceMgr = resourceManager;
}
@@ -67,6 +71,8 @@ public class KVMFencerTest {
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
Mockito.when(host.getId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -80,6 +86,8 @@ public class KVMFencerTest {
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getStatus()).thenReturn(Status.Down);
Mockito.when(host.getId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -94,12 +102,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+ Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+ Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -118,12 +130,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+ Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+ Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -141,12 +157,16 @@ public class KVMFencerTest {
Mockito.when(host.getClusterId()).thenReturn(1l);
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(host.getStatus()).thenReturn(Status.Up);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(1l);
HostVO secondHost = Mockito.mock(HostVO.class);
Mockito.when(secondHost.getClusterId()).thenReturn(1l);
Mockito.when(secondHost.getHypervisorType()).thenReturn(HypervisorType.KVM);
Mockito.when(secondHost.getStatus()).thenReturn(Status.Up);
+ Mockito.when(secondHost.getDataCenterId()).thenReturn(1l);
+ Mockito.when(secondHost.getPodId()).thenReturn(1l);
Mockito.when(host.getId()).thenReturn(2l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
@@ -165,6 +185,8 @@ public class KVMFencerTest {
Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.Any);
Mockito.when(host.getStatus()).thenReturn(Status.Down);
Mockito.when(host.getId()).thenReturn(1l);
+ Mockito.when(host.getDataCenterId()).thenReturn(1l);
+ Mockito.when(host.getPodId()).thenReturn(1l);
VirtualMachine virtualMachine = Mockito.mock(VirtualMachine.class);
Mockito.when(resourceManager.listAllHostsInCluster(1l)).thenReturn(Collections.singletonList(host));
[2/2] git commit: updated refs/heads/master to fbe3b04
Posted by wi...@apache.org.
CLOUDSTACK-8628: kvm: Disable Fencing when no NFS storage pools are present
On NFS we write a heartbeat, but without those we can not safely
fence off a host.
If we fence without knowing about a heartbeat we can cause a split-brain situation.
Signed-off-by: Wido den Hollander <wi...@widodh.nl>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/d1f76a2a
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/d1f76a2a
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/d1f76a2a
Branch: refs/heads/master
Commit: d1f76a2a84c531a73285630c4f6283dc17143f82
Parents: 8d96aab
Author: Wido den Hollander <wi...@widodh.nl>
Authored: Mon Jul 13 17:26:00 2015 +0200
Committer: Wido den Hollander <wi...@widodh.nl>
Committed: Fri Jul 17 12:33:11 2015 +0200
----------------------------------------------------------------------
.../resource/wrapper/LibvirtFenceCommandWrapper.java | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d1f76a2a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java
index 8fdf46b..cb313df 100644
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java
@@ -48,6 +48,18 @@ public final class LibvirtFenceCommandWrapper extends CommandWrapper<FenceComman
final KVMHAMonitor monitor = libvirtComputingResource.getMonitor();
final List<NfsStoragePool> pools = monitor.getStoragePools();
+
+ /**
+ * We can only safely fence off hosts when we use NFS
+ * On NFS primary storage pools hosts continuesly write
+ * a heartbeat. Disable Fencing Off for hosts without NFS
+ */
+ if (pools.size() == 0) {
+ String logline = "No NFS storage pools found. No way to safely fence " + command.getVmName() + " on host " + command.getHostGuid();
+ s_logger.warn(logline);
+ return new FenceAnswer(command, false, logline);
+ }
+
final KVMHAChecker ha = new KVMHAChecker(pools, command.getHostIp());
final Future<Boolean> future = executors.submit(ha);