You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ga...@apache.org on 2019/01/24 22:18:13 UTC
[cloudstack] branch master updated: kvm: Properly report available
memory to Management Server (#2795)
This is an automated email from the ASF dual-hosted git repository.
gabriel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/master by this push:
new c496c84 kvm: Properly report available memory to Management Server (#2795)
c496c84 is described below
commit c496c84c6c727a84862cbbe2d870ff57939488b4
Author: Wido den Hollander <wi...@widodh.nl>
AuthorDate: Thu Jan 24 23:18:04 2019 +0100
kvm: Properly report available memory to Management Server (#2795)
The KVM Agent had two mechanisms for reporting its capabilities
and memory to the Management Server.
On startup it would ask libvirt the amount of Memory the Host has
and subtract and add the reserved and overcommit memory.
When the HostStats were however reported to the Management Server
these two configured values on the Agent were no longer reported
in the statistics thus showing all the available memory in the
Agent/Host to the Management Server.
This commit unifies this by using the same logic on Agent Startup
and during statistics reporting.
memory=3069636608, reservedMemory=1073741824
This was reported by a 4GB Hypervisor with this setting:
host.reserved.mem.mb=1024
The GUI (thus API) would then show:
Memory Total 2.86 GB
This way the Agent properly 'lies' to the Management Server about its
capabilities in terms of Memory.
This is very helpful if you want to overprovision or undercommit machines
for various reasons.
Overcommitting can be done when KSM or ZSwap or a fast SWAP device is
installed in the machine.
Underprovisioning is done when the Host might run other tasks then a KVM
hypervisor, for example when it runs in a hyperconverged setup with Ceph.
In addition internally many values have been changed from a Double to a Long
and also store the amount of bytes instead of Kilobytes.
Signed-off-by: Wido den Hollander <wi...@widodh.nl>
---
.../kvm/resource/LibvirtComputingResource.java | 82 ++-----------
.../wrapper/LibvirtGetHostStatsCommandWrapper.java | 5 +-
.../apache/cloudstack/utils/linux/KVMHostInfo.java | 132 +++++++++++++++++++++
.../org/apache/cloudstack/utils/linux/MemStat.java | 33 ++++--
.../kvm/resource/LibvirtComputingResourceTest.java | 11 +-
.../cloudstack/utils/linux/KVMHostInfoTest.java | 37 ++++++
.../apache/cloudstack/utils/linux/MemStatTest.java | 40 +++++--
7 files changed, 233 insertions(+), 107 deletions(-)
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index b21cd78..2e799b1 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -19,9 +19,7 @@ package com.cloud.hypervisor.kvm.resource;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
-import java.io.FileReader;
import java.io.IOException;
-import java.io.Reader;
import java.io.StringReader;
import java.net.InetAddress;
import java.net.URI;
@@ -55,12 +53,12 @@ import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.hypervisor.HypervisorUtils;
import org.apache.cloudstack.utils.linux.CPUStat;
+import org.apache.cloudstack.utils.linux.KVMHostInfo;
import org.apache.cloudstack.utils.linux.MemStat;
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.log4j.Logger;
@@ -311,8 +309,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
protected int _cmdsTimeout;
protected int _stopTimeout;
protected CPUStat _cpuStat = new CPUStat();
- protected MemStat _memStat = new MemStat();
-
+ protected MemStat _memStat = new MemStat(_dom0MinMem, _dom0OvercommitMem);
private final LibvirtUtilitiesHelper libvirtUtilitiesHelper = new LibvirtUtilitiesHelper();
@Override
@@ -871,7 +868,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
value = (String)params.get("host.reserved.mem.mb");
// Reserve 1GB unless admin overrides
- _dom0MinMem = NumbersUtil.parseInt(value, 1024) * 1024 * 1024L;
+ _dom0MinMem = NumbersUtil.parseInt(value, 1024) * 1024* 1024L;
value = (String)params.get("host.overcommit.mem.mb");
// Support overcommit memory for host if host uses ZSWAP, KSM and other memory
@@ -2661,12 +2658,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
@Override
public StartupCommand[] initialize() {
- final List<Object> info = getHostInfo();
+ final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem);
+
+ final String capabilities = String.join(",", info.getCapabilities());
final StartupRoutingCommand cmd =
- new StartupRoutingCommand((Integer)info.get(0), (Long)info.get(1), (Long)info.get(2), (Long)info.get(4), (String)info.get(3), _hypervisorType,
+ new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType,
RouterPrivateIpStrategy.HostLocal);
- cmd.setCpuSockets((Integer)info.get(5));
+ cmd.setCpuSockets(info.getCpuSockets());
fillNetworkInformation(cmd);
_privateIp = cmd.getPrivateIpAddress();
cmd.getHostDetails().putAll(getVersionStrings());
@@ -2886,71 +2885,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return vmStates;
}
- protected List<Object> getHostInfo() {
- final ArrayList<Object> info = new ArrayList<Object>();
- long speed = 0;
- long cpus = 0;
- long ram = 0;
- int cpuSockets = 0;
- String cap = null;
- try {
- final Connect conn = LibvirtConnection.getConnection();
- final NodeInfo hosts = conn.nodeInfo();
- speed = getCpuSpeed(hosts);
-
- /*
- * Some CPUs report a single socket and multiple NUMA cells.
- * We need to multiply them to get the correct socket count.
- */
- cpuSockets = hosts.sockets;
- if (hosts.nodes > 0) {
- cpuSockets = hosts.sockets * hosts.nodes;
- }
- cpus = hosts.cpus;
- ram = hosts.memory * 1024L;
- final LibvirtCapXMLParser parser = new LibvirtCapXMLParser();
- parser.parseCapabilitiesXML(conn.getCapabilities());
- final ArrayList<String> oss = parser.getGuestOsType();
- for (final String s : oss) {
- /*
- * Even host supports guest os type more than hvm, we only
- * report hvm to management server
- */
- if (s.equalsIgnoreCase("hvm")) {
- cap = "hvm";
- }
- }
- } catch (final LibvirtException e) {
- s_logger.trace("Ignoring libvirt error.", e);
- }
-
- if (isSnapshotSupported()) {
- cap = cap + ",snapshot";
- }
-
- info.add((int)cpus);
- info.add(speed);
- // Report system's RAM as actual RAM minus host OS reserved RAM
- ram = ram - _dom0MinMem + _dom0OvercommitMem;
- info.add(ram);
- info.add(cap);
- info.add(_dom0MinMem);
- info.add(cpuSockets);
- s_logger.debug("cpus=" + cpus + ", speed=" + speed + ", ram=" + ram + ", _dom0MinMem=" + _dom0MinMem + ", _dom0OvercommitMem=" + _dom0OvercommitMem + ", cpu sockets=" + cpuSockets);
-
- return info;
- }
-
- protected static long getCpuSpeed(final NodeInfo nodeInfo) {
- try (final Reader reader = new FileReader(
- "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq")) {
- return Long.parseLong(IOUtils.toString(reader).trim()) / 1000;
- } catch (IOException | NumberFormatException e) {
- s_logger.warn("Could not read cpuinfo_max_freq");
- return nodeInfo.mhz;
- }
- }
-
public String rebootVM(final Connect conn, final String vmName) {
Domain dm = null;
String msg = null;
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetHostStatsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetHostStatsCommandWrapper.java
index 80bd5fa..b04a866 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetHostStatsCommandWrapper.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetHostStatsCommandWrapper.java
@@ -42,13 +42,10 @@ public final class LibvirtGetHostStatsCommandWrapper extends CommandWrapper<GetH
MemStat memStat = libvirtComputingResource.getMemStat();
final double cpuUtil = cpuStat.getCpuUsedPercent();
- memStat.refresh();
- double totMem = memStat.getTotal();
- double freeMem = memStat.getAvailable();
final Pair<Double, Double> nicStats = libvirtComputingResource.getNicStats(libvirtComputingResource.getPublicBridgeName());
- final HostStatsEntry hostStats = new HostStatsEntry(command.getHostId(), cpuUtil, nicStats.first() / 1024, nicStats.second() / 1024, "host", totMem, freeMem, 0, 0);
+ final HostStatsEntry hostStats = new HostStatsEntry(command.getHostId(), cpuUtil, nicStats.first() / 1024, nicStats.second() / 1024, "host", memStat.getTotal() / 1024, memStat.getAvailable() / 1024, 0, 0);
return new GetHostStatsAnswer(command, hostStats);
}
}
\ No newline at end of file
diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java
new file mode 100644
index 0000000..1f28304
--- /dev/null
+++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.utils.linux;
+
+import com.cloud.hypervisor.kvm.resource.LibvirtCapXMLParser;
+import com.cloud.hypervisor.kvm.resource.LibvirtConnection;
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
+import org.libvirt.Connect;
+import org.libvirt.LibvirtException;
+import org.libvirt.NodeInfo;
+
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+
+public class KVMHostInfo {
+
+ private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class);
+
+ private int cpus;
+ private int cpusockets;
+ private long cpuSpeed;
+ private long totalMemory;
+ private long reservedMemory;
+ private long overCommitMemory;
+ private List<String> capabilities = new ArrayList<>();
+
+ public KVMHostInfo(long reservedMemory, long overCommitMemory) {
+ this.reservedMemory = reservedMemory;
+ this.overCommitMemory = overCommitMemory;
+ this.getHostInfoFromLibvirt();
+ this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal();
+ }
+
+ public int getCpus() {
+ return this.cpus;
+ }
+
+ public int getCpuSockets() {
+ return this.cpusockets;
+ }
+
+ public long getCpuSpeed() {
+ return this.cpuSpeed;
+ }
+
+ public long getTotalMemory() {
+ return this.totalMemory;
+ }
+
+ public long getReservedMemory() {
+ return this.reservedMemory;
+ }
+
+ public long getOverCommitMemory() {
+ return this.overCommitMemory;
+ }
+
+ public List<String> getCapabilities() {
+ return this.capabilities;
+ }
+
+ protected static long getCpuSpeed(final NodeInfo nodeInfo) {
+ try (final Reader reader = new FileReader(
+ "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq")) {
+ return Long.parseLong(IOUtils.toString(reader).trim()) / 1000;
+ } catch (IOException | NumberFormatException e) {
+ LOGGER.info("Could not read cpuinfo_max_freq, falling back on libvirt");
+ return nodeInfo.mhz;
+ }
+ }
+
+ private void getHostInfoFromLibvirt() {
+ try {
+ final Connect conn = LibvirtConnection.getConnection();
+ final NodeInfo hosts = conn.nodeInfo();
+ this.cpuSpeed = getCpuSpeed(hosts);
+
+ /*
+ * Some CPUs report a single socket and multiple NUMA cells.
+ * We need to multiply them to get the correct socket count.
+ */
+ this.cpusockets = hosts.sockets;
+ if (hosts.nodes > 0) {
+ this.cpusockets = hosts.sockets * hosts.nodes;
+ }
+ this.cpus = hosts.cpus;
+
+ final LibvirtCapXMLParser parser = new LibvirtCapXMLParser();
+ parser.parseCapabilitiesXML(conn.getCapabilities());
+ final ArrayList<String> oss = parser.getGuestOsType();
+ for (final String s : oss) {
+ /*
+ * Even host supports guest os type more than hvm, we only
+ * report hvm to management server
+ */
+ String hvmCapability = "hvm";
+ if (s.equalsIgnoreCase(hvmCapability)) {
+ if (!this.capabilities.contains(hvmCapability)) {
+ this.capabilities.add(hvmCapability);
+ }
+ }
+ }
+
+ /*
+ Any modern Qemu/KVM supports snapshots
+ We used to check if this was supported, but that is no longer required
+ */
+ this.capabilities.add("snapshot");
+ conn.close();
+ } catch (final LibvirtException e) {
+ LOGGER.error("Caught libvirt exception while fetching host information", e);
+ }
+ }
+}
diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/MemStat.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/MemStat.java
index 2029af3..afcdf97 100644
--- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/MemStat.java
+++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/MemStat.java
@@ -22,30 +22,47 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Scanner;
+
public class MemStat {
+ /*
+ Gather Memory Statistics of the current node by opening /proc/meminfo
+ which contains the memory information in KiloBytes.
+
+ Convert this all to bytes and return Long as a type with the information
+ in bytes
+ */
protected final static String MEMINFO_FILE = "/proc/meminfo";
protected final static String FREE_KEY = "MemFree";
protected final static String CACHE_KEY = "Cached";
protected final static String TOTAL_KEY = "MemTotal";
+ long reservedMemory;
+ long overCommitMemory;
- private final Map<String, Double> _memStats = new HashMap<String, Double>();
+ private final Map<String, Long> _memStats = new HashMap<>();
public MemStat() {
+ this(0,0);
+ }
+
+ public MemStat(long reservedMemory, long overCommitMemory) {
+ this.reservedMemory = reservedMemory;
+ this.overCommitMemory = overCommitMemory;
+ this.refresh();
}
- public Double getTotal() {
- return _memStats.get(TOTAL_KEY);
+ public long getTotal() {
+ return _memStats.get(TOTAL_KEY) - reservedMemory + overCommitMemory;
}
- public Double getAvailable() {
+ public long getAvailable() {
return getFree() + getCache();
}
- public Double getFree() {
- return _memStats.get(FREE_KEY);
+ public long getFree() {
+ return _memStats.get(FREE_KEY) - reservedMemory + overCommitMemory;
}
- public Double getCache() {
+ public long getCache() {
return _memStats.get(CACHE_KEY);
}
@@ -63,7 +80,7 @@ public class MemStat {
while(scanner.hasNext()) {
String[] stats = scanner.next().split("\\:\\s+");
if (stats.length == 2) {
- _memStats.put(stats[0], Double.valueOf(stats[1].replaceAll("\\s+\\w+","")));
+ _memStats.put(stats[0], Long.valueOf(stats[1].replaceAll("\\s+\\w+","")) * 1024L);
}
}
}
diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java
index 69954d0..0a5c3d6 100644
--- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java
@@ -534,13 +534,6 @@ public class LibvirtComputingResourceTest {
Assert.assertTrue(vmStat.getTargetMemoryKBs() >= vmStat.getMemoryKBs());
}
- @Test
- public void getCpuSpeed() {
- Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
- final NodeInfo nodeInfo = Mockito.mock(NodeInfo.class);
- LibvirtComputingResource.getCpuSpeed(nodeInfo);
- }
-
/*
* New Tests
*/
@@ -988,8 +981,8 @@ public class LibvirtComputingResourceTest {
when(libvirtComputingResource.getMemStat()).thenReturn(memStat);
when(libvirtComputingResource.getNicStats(Mockito.anyString())).thenReturn(new Pair<Double, Double>(1.0d, 1.0d));
when(cpuStat.getCpuUsedPercent()).thenReturn(0.5d);
- when(memStat.getAvailable()).thenReturn(1500.5d);
- when(memStat.getTotal()).thenReturn(15000d);
+ when(memStat.getAvailable()).thenReturn(1500L);
+ when(memStat.getTotal()).thenReturn(15000L);
final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance();
assertNotNull(wrapper);
diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java
new file mode 100644
index 0000000..bc6b01c
--- /dev/null
+++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.utils.linux;
+
+import org.apache.commons.lang.SystemUtils;
+
+import org.hamcrest.Matchers;
+import org.junit.Test;
+import org.junit.Assume;
+import org.junit.Assert;
+import org.mockito.Mockito;
+
+import org.libvirt.NodeInfo;
+
+public class KVMHostInfoTest {
+ @Test
+ public void getCpuSpeed() {
+ Assume.assumeTrue(SystemUtils.IS_OS_LINUX);
+ NodeInfo nodeInfo = Mockito.mock(NodeInfo.class);
+ nodeInfo.mhz = 1000;
+ Assert.assertThat(KVMHostInfo.getCpuSpeed(nodeInfo), Matchers.greaterThan(0l));
+ }
+}
\ No newline at end of file
diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/MemStatTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/MemStatTest.java
index 5e77606..476dc76 100644
--- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/MemStatTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/MemStatTest.java
@@ -22,16 +22,16 @@ import org.junit.Test;
import java.util.Scanner;
public class MemStatTest {
+ final String memInfo = "MemTotal: 5830236 kB\n" +
+ "MemFree: 156752 kB\n" +
+ "Buffers: 326836 kB\n" +
+ "Cached: 2606764 kB\n" +
+ "SwapCached: 0 kB\n" +
+ "Active: 4260808 kB\n" +
+ "Inactive: 949392 kB\n";
+
@Test
public void getMemInfoParseTest() {
- String memInfo = "MemTotal: 5830236 kB\n" +
- "MemFree: 156752 kB\n" +
- "Buffers: 326836 kB\n" +
- "Cached: 2606764 kB\n" +
- "SwapCached: 0 kB\n" +
- "Active: 4260808 kB\n" +
- "Inactive: 949392 kB\n";
-
MemStat memStat = null;
try {
memStat = new MemStat();
@@ -46,9 +46,25 @@ public class MemStatTest {
Scanner scanner = new Scanner(memInfo);
memStat.parseFromScanner(scanner);
- Assert.assertEquals(memStat.getTotal(), Double.valueOf(5830236));
- Assert.assertEquals(memStat.getAvailable(), Double.valueOf(2763516));
- Assert.assertEquals(memStat.getFree(), Double.valueOf(156752));
- Assert.assertEquals(memStat.getCache(), Double.valueOf(2606764));
+ Assert.assertEquals(memStat.getTotal(), 5970161664L);
+ Assert.assertEquals(memStat.getAvailable(), 2829840384L);
+ Assert.assertEquals(memStat.getFree(), 160514048L);
+ Assert.assertEquals(memStat.getCache(), 2669326336L);
+ }
+
+ @Test
+ public void reservedMemoryTest() {
+ MemStat memStat = null;
+ try {
+ memStat = new MemStat(1024, 2048);
+ } catch (RuntimeException ex) {
+ if (memStat == null) {
+ throw ex;
+ }
+ }
+ Scanner scanner = new Scanner(memInfo);
+ memStat.parseFromScanner(scanner);
+
+ Assert.assertEquals(memStat.getTotal(), 5970162688L);
}
}