You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by bh...@apache.org on 2015/09/02 11:25:20 UTC
[1/2] git commit: updated refs/heads/4.5 to 047d759
Repository: cloudstack
Updated Branches:
refs/heads/4.5 a6c630c54 -> 047d759a7
CLOUDSTACK-8762: Confirm disk activity before starting a VM
Implements a VM volume/disk file activity checker that checks if QCOW2 file
has been changed before starting the VM. This is useful as a pessimistic
approach to save VMs that were running on faulty hosts that CloudStack could
try to launch on other hosts while the host was not cleanly fenced. This is
optional and available only if you enable the settings in agent.properties
file, on per-host basis.
Signed-off-by: Rohit Yadav <ro...@shapeblue.com>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/711acfa9
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/711acfa9
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/711acfa9
Branch: refs/heads/4.5
Commit: 711acfa93dd8fcf1083062753464901c171f4d6f
Parents: e967df9
Author: Rohit Yadav <ro...@shapeblue.com>
Authored: Fri Aug 28 15:52:06 2015 +0530
Committer: Rohit Yadav <ro...@shapeblue.com>
Committed: Fri Aug 28 15:52:06 2015 +0530
----------------------------------------------------------------------
agent/conf/agent.properties | 16 +++
.../kvm/resource/LibvirtComputingResource.java | 31 ++++++
.../utils/hypervisor/HypervisorUtils.java | 68 ++++++++++++
.../utils/hypervisor/HypervisorUtilsTest.java | 111 +++++++++++++++++++
4 files changed, 226 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/711acfa9/agent/conf/agent.properties
----------------------------------------------------------------------
diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties
index fcd8b5c..daad05f 100644
--- a/agent/conf/agent.properties
+++ b/agent/conf/agent.properties
@@ -133,6 +133,22 @@ hypervisor.type=kvm
# Disable memory ballooning on vm guests for overcommit, by default overcommit
# feature enables balloon and sets currentMemory to a minimum value.
#
+# vm.diskactivity.checkenabled=false
+# Set to true to check disk activity on VM's disks before starting a VM. This only applies
+# to QCOW2 files, and ensures that there is no other running instance accessing
+# the file before starting. It works by checking the modify time against the current time,
+# so care must be taken to ensure the cluster has time synced, otherwise VMs may fail to start.
+#
+# vm.diskactivity.checktimeout_s=120
+# Timeout for giving up on waiting for VM's disk files to become inactive. Hitting
+# this timeout will result in failure to start VM.
+#
+# vm.diskactivity.inactivetime_ms=30000
+# This is the length of time that the disk needs to be inactive in order to pass the check.
+# This means current time minus mtime of disk file needs to be greater than this number.
+# It also has the side effect of setting the minimum threshold between a stop and start of
+# a given VM.
+#
# kvmclock.disable=false
# Some newer linux kernels are incapable of reliably migrating vms with kvmclock
# This is a workaround for the bug, admin can set this to true per-host
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/711acfa9/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index ae48783..0f5908f 100755
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -91,6 +91,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.cloudstack.utils.hypervisor.HypervisorUtils;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.AttachIsoCommand;
@@ -319,6 +320,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
private int _migrateSpeed;
private int _migrateDowntime;
private int _migratePauseAfter;
+ protected boolean _diskActivityCheckEnabled;
+ protected long _diskActivityCheckFileSizeMin = 10485760; // 10MB
+ protected int _diskActivityCheckTimeoutSeconds = 120; // 120s
+ protected long _diskActivityInactiveThresholdMilliseconds = 30000; // 30s
private long _hvVersion;
private long _kernelVersion;
@@ -969,6 +974,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
params.put("libvirtVersion", _hypervisorLibvirtVersion);
configureVifDrivers(params);
+ configureDiskActivityChecks(params);
KVMStorageProcessor storageProcessor = new KVMStorageProcessor(_storagePoolMgr, this);
storageProcessor.configure(name, params);
@@ -982,6 +988,20 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return true;
}
+ protected void configureDiskActivityChecks(Map<String, Object> params) {
+ _diskActivityCheckEnabled = Boolean.parseBoolean((String)params.get("vm.diskactivity.checkenabled"));
+ if (_diskActivityCheckEnabled) {
+ int timeout = NumbersUtil.parseInt((String)params.get("vm.diskactivity.checktimeout_s"), 0);
+ if (timeout > 0) {
+ _diskActivityCheckTimeoutSeconds = timeout;
+ }
+ long inactiveTime = NumbersUtil.parseLong((String)params.get("vm.diskactivity.inactivetime_ms"), 0L);
+ if (inactiveTime > 0) {
+ _diskActivityInactiveThresholdMilliseconds = inactiveTime;
+ }
+ }
+ }
+
protected void configureVifDrivers(Map<String, Object> params) throws ConfigurationException {
final String LIBVIRT_VIF_DRIVER = "libvirt.vif.driver";
@@ -3923,6 +3943,17 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
volPath = physicalDisk.getPath();
}
+ // check for disk activity, if detected we should exit because vm is running elsewhere
+ if (_diskActivityCheckEnabled && physicalDisk != null && physicalDisk.getFormat() == PhysicalDiskFormat.QCOW2) {
+ s_logger.debug("Checking physical disk file at path " + volPath + " for disk activity to ensure vm is not running elsewhere");
+ try {
+ HypervisorUtils.checkVolumeFileForActivity(volPath, _diskActivityCheckTimeoutSeconds, _diskActivityInactiveThresholdMilliseconds, _diskActivityCheckFileSizeMin);
+ } catch (IOException ex) {
+ throw new CloudRuntimeException("Unable to check physical disk file for activity", ex);
+ }
+ s_logger.debug("Disk activity check cleared");
+ }
+
// if params contains a rootDiskController key, use its value (this is what other HVs are doing)
DiskDef.diskBus diskBusType = null;
Map <String, String> params = vmSpec.getDetails();
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/711acfa9/utils/src/org/apache/cloudstack/utils/hypervisor/HypervisorUtils.java
----------------------------------------------------------------------
diff --git a/utils/src/org/apache/cloudstack/utils/hypervisor/HypervisorUtils.java b/utils/src/org/apache/cloudstack/utils/hypervisor/HypervisorUtils.java
new file mode 100644
index 0000000..a0a2093
--- /dev/null
+++ b/utils/src/org/apache/cloudstack/utils/hypervisor/HypervisorUtils.java
@@ -0,0 +1,68 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.utils.hypervisor;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.concurrent.TimeUnit;
+
+public class HypervisorUtils {
+ public static final Logger s_logger = Logger.getLogger(HypervisorUtils.class);
+
+ public static void checkVolumeFileForActivity(final String filePath, int timeoutSeconds, long inactiveThresholdMilliseconds, long minimumFileSize) throws IOException {
+ File file = new File(filePath);
+ if (!file.exists()) {
+ throw new CloudRuntimeException("File " + file.getAbsolutePath() + " not found");
+ }
+ if (file.length() < minimumFileSize) {
+ s_logger.debug("VM disk file too small, fresh clone? skipping modify check");
+ return;
+ }
+ int waitedSeconds = 0;
+ int intervalSeconds = 1;
+ while (true) {
+ BasicFileAttributes attrs = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
+ long modifyIdle = System.currentTimeMillis() - attrs.lastModifiedTime().toMillis();
+ long accessIdle = System.currentTimeMillis() - attrs.lastAccessTime().toMillis();
+ if (modifyIdle > inactiveThresholdMilliseconds && accessIdle > inactiveThresholdMilliseconds) {
+ s_logger.debug("File " + filePath + " has not been accessed or modified for at least " + inactiveThresholdMilliseconds + " ms");
+ return;
+ } else {
+ s_logger.debug("File was modified " + modifyIdle + "ms ago, accessed " + accessIdle + "ms ago, waiting for inactivity threshold of "
+ + inactiveThresholdMilliseconds + "ms or timeout of " + timeoutSeconds + "s (waited " + waitedSeconds + "s)");
+ }
+ try {
+ TimeUnit.SECONDS.sleep(intervalSeconds);
+ } catch (InterruptedException ex) {
+ throw new CloudRuntimeException("Interrupted while waiting for activity on " + filePath + " to subside", ex);
+ }
+ waitedSeconds += intervalSeconds;
+ if (waitedSeconds >= timeoutSeconds) {
+ throw new CloudRuntimeException("Reached timeout while waiting for activity on " + filePath + " to subside");
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/711acfa9/utils/test/org/apache/cloudstack/utils/hypervisor/HypervisorUtilsTest.java
----------------------------------------------------------------------
diff --git a/utils/test/org/apache/cloudstack/utils/hypervisor/HypervisorUtilsTest.java b/utils/test/org/apache/cloudstack/utils/hypervisor/HypervisorUtilsTest.java
new file mode 100644
index 0000000..b54f4cb
--- /dev/null
+++ b/utils/test/org/apache/cloudstack/utils/hypervisor/HypervisorUtilsTest.java
@@ -0,0 +1,111 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.utils.hypervisor;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+
+@RunWith(MockitoJUnitRunner.class)
+public class HypervisorUtilsTest {
+
+ final long _minFileSize = 10485760L;
+
+ @Test
+ public void checkVolumeFileForActivitySmallFileTest() throws IOException {
+ System.out.print("Testing don't block on newly created clones - ");
+ String filePath = "./testsmallfileinactive";
+ int timeoutSeconds = 10;
+ long thresholdMilliseconds = 5000;
+ File file = new File(filePath);
+ setupcheckVolumeFileForActivityFile(file, 0);
+
+ long startTime = System.currentTimeMillis();
+ HypervisorUtils.checkVolumeFileForActivity(filePath, timeoutSeconds, thresholdMilliseconds, _minFileSize);
+ long endTime = System.currentTimeMillis();
+
+ Assert.assertEquals(startTime, endTime, 1000L);
+ System.out.println("pass");
+
+ file.delete();
+ }
+
+ @Test
+ public void checkVolumeFileForActivityTest() throws IOException {
+ System.out.print("Testing block on modified files - ");
+ String filePath = "./testfileinactive";
+ int timeoutSeconds = 5;
+ long thresholdMilliseconds = 2000;
+ File file = new File(filePath);
+ setupcheckVolumeFileForActivityFile(file, _minFileSize);
+
+ long startTime = System.currentTimeMillis();
+ org.apache.cloudstack.utils.hypervisor.HypervisorUtils.checkVolumeFileForActivity(filePath, timeoutSeconds, thresholdMilliseconds, _minFileSize);
+ long duration = System.currentTimeMillis() - startTime;
+
+ Assert.assertFalse("Didn't block long enough, expected at least " + thresholdMilliseconds + " and got " + duration, duration < thresholdMilliseconds);
+ System.out.println("pass");
+
+ file.delete();
+ }
+
+ @Test(expected=CloudRuntimeException.class)
+ public void checkVolumeFileForActivityTimeoutTest() throws IOException {
+ System.out.print("Testing timeout of blocking on modified files - ");
+ String filePath = "./testfileinactive";
+ int timeoutSeconds = 3;
+ long thresholdMilliseconds = 5000;
+ File file = new File(filePath);
+ setupcheckVolumeFileForActivityFile(file, _minFileSize);
+
+ try {
+ HypervisorUtils.checkVolumeFileForActivity(filePath, timeoutSeconds, thresholdMilliseconds, _minFileSize);
+ } catch (CloudRuntimeException ex) {
+ System.out.println("pass");
+ throw ex;
+ } finally {
+ file.delete();
+ }
+ System.out.println("Fail");
+ }
+
+ private void setupcheckVolumeFileForActivityFile(File file, long minSize) throws IOException {
+ if (file.exists()) {
+ file.delete();
+ }
+ file.createNewFile();
+ char[] chars = new char[1048576];
+ Arrays.fill(chars, 'X');
+ long written = 0;
+ FileWriter writer = new FileWriter(file);
+ while (written < minSize) {
+ writer.write(chars);
+ written += chars.length;
+ }
+ writer.close();
+ }
+}
[2/2] git commit: updated refs/heads/4.5 to 047d759
Posted by bh...@apache.org.
Merge pull request #754 from shapeblue/4.5-CLOUDSTACK-8762
CLOUDSTACK-8762: Check to confirm disk activity before starting a VMImplements a VM volume/disk file activity checker that checks if QCOW2 file
has been changed before starting the VM. This is useful as a pessimistic
approach to save VMs that were running on faulty hosts that CloudStack could
try to launch on other hosts while the host was not cleanly fenced. This is
optional and available only if you enable the settings in agent.properties
file, on per-host basis.
Signed-off-by: Rohit Yadav <ro...@shapeblue.com>
* pr/754:
CLOUDSTACK-8762: Confirm disk activity before starting a VM
Signed-off-by: Rohit Yadav <ro...@shapeblue.com>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/047d759a
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/047d759a
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/047d759a
Branch: refs/heads/4.5
Commit: 047d759a73572476353dc47d0ce41e8443829a4a
Parents: a6c630c 711acfa
Author: Rohit Yadav <ro...@shapeblue.com>
Authored: Wed Sep 2 14:54:58 2015 +0530
Committer: Rohit Yadav <ro...@shapeblue.com>
Committed: Wed Sep 2 14:54:58 2015 +0530
----------------------------------------------------------------------
agent/conf/agent.properties | 16 +++
.../kvm/resource/LibvirtComputingResource.java | 31 ++++++
.../utils/hypervisor/HypervisorUtils.java | 68 ++++++++++++
.../utils/hypervisor/HypervisorUtilsTest.java | 111 +++++++++++++++++++
4 files changed, 226 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/047d759a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
----------------------------------------------------------------------