You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/06/28 21:29:27 UTC
[12/50] [abbrv] hadoop git commit: YARN-8444: NodeResourceMonitor
crashes on bad swapFree value. Contributed by Jim Brennan.
YARN-8444: NodeResourceMonitor crashes on bad swapFree value. Contributed by Jim Brennan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64321286
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64321286
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64321286
Branch: refs/heads/HDDS-48
Commit: 6432128622d64f3f9dd638b9c254c77cdf5408aa
Parents: 30728ac
Author: Eric E Payne <er...@oath.com>
Authored: Fri Jun 22 17:15:29 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Fri Jun 22 17:15:29 2018 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/util/SysInfoLinux.java | 19 ++++++-
.../apache/hadoop/util/TestSysInfoLinux.java | 60 ++++++++++++++++++++
2 files changed, 77 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
index 7fd1990..2c2aca3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
@@ -217,6 +217,21 @@ public class SysInfoLinux extends SysInfo {
}
/**
+ *
+ * Wrapper for Long.parseLong() that returns zero if the value is
+ * invalid. Under some circumstances, swapFree in /proc/meminfo can
+ * go negative, reported as a very large decimal value.
+ */
+ private long safeParseLong(String strVal) {
+ long parsedVal;
+ try {
+ parsedVal = Long.parseLong(strVal);
+ } catch (NumberFormatException nfe) {
+ parsedVal = 0;
+ }
+ return parsedVal;
+ }
+ /**
* Read /proc/meminfo, parse and compute memory information.
* @param readAgain if false, read only on the first time
*/
@@ -252,9 +267,9 @@ public class SysInfoLinux extends SysInfo {
} else if (mat.group(1).equals(SWAPTOTAL_STRING)) {
swapSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(MEMFREE_STRING)) {
- ramSizeFree = Long.parseLong(mat.group(2));
+ ramSizeFree = safeParseLong(mat.group(2));
} else if (mat.group(1).equals(SWAPFREE_STRING)) {
- swapSizeFree = Long.parseLong(mat.group(2));
+ swapSizeFree = safeParseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVE_STRING)) {
inactiveSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVEFILE_STRING)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
index a646a41..0ae5d3c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
@@ -161,6 +161,36 @@ public class TestSysInfoLinux {
"DirectMap2M: 2027520 kB\n" +
"DirectMap1G: 132120576 kB\n";
+ static final String MEMINFO_FORMAT3 =
+ "MemTotal: %d kB\n" +
+ "MemFree: %s kB\n" +
+ "Buffers: 138244 kB\n" +
+ "Cached: 947780 kB\n" +
+ "SwapCached: 142880 kB\n" +
+ "Active: 3229888 kB\n" +
+ "Inactive: %d kB\n" +
+ "SwapTotal: %d kB\n" +
+ "SwapFree: %s kB\n" +
+ "Dirty: 122012 kB\n" +
+ "Writeback: 0 kB\n" +
+ "AnonPages: 2710792 kB\n" +
+ "Mapped: 24740 kB\n" +
+ "Slab: 132528 kB\n" +
+ "SReclaimable: 105096 kB\n" +
+ "SUnreclaim: 27432 kB\n" +
+ "PageTables: 11448 kB\n" +
+ "NFS_Unstable: 0 kB\n" +
+ "Bounce: 0 kB\n" +
+ "CommitLimit: 4125904 kB\n" +
+ "Committed_AS: 4143556 kB\n" +
+ "VmallocTotal: 34359738367 kB\n" +
+ "VmallocUsed: 1632 kB\n" +
+ "VmallocChunk: 34359736375 kB\n" +
+ "HugePages_Total: %d\n" +
+ "HugePages_Free: 0\n" +
+ "HugePages_Rsvd: 0\n" +
+ "Hugepagesize: 2048 kB";
+
static final String CPUINFO_FORMAT =
"processor : %s\n" +
"vendor_id : AuthenticAMD\n" +
@@ -384,6 +414,36 @@ public class TestSysInfoLinux {
(nrHugePages * 2048) + swapTotal));
}
+ /**
+ * Test parsing /proc/meminfo
+ * @throws IOException
+ */
+ @Test
+ public void parsingProcMemFileWithBadValues() throws IOException {
+ long memTotal = 4058864L;
+ long memFree = 0L; // bad value should return 0
+ long inactive = 567732L;
+ long swapTotal = 2096472L;
+ long swapFree = 0L; // bad value should return 0
+ int nrHugePages = 10;
+ String badFreeValue = "18446744073709551596";
+ File tempFile = new File(FAKE_MEMFILE);
+ tempFile.deleteOnExit();
+ FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
+ fWriter.write(String.format(MEMINFO_FORMAT3,
+ memTotal, badFreeValue, inactive, swapTotal, badFreeValue, nrHugePages));
+
+ fWriter.close();
+ assertEquals(plugin.getAvailablePhysicalMemorySize(),
+ 1024L * (memFree + inactive));
+ assertEquals(plugin.getAvailableVirtualMemorySize(),
+ 1024L * (memFree + inactive + swapFree));
+ assertEquals(plugin.getPhysicalMemorySize(),
+ 1024L * (memTotal - (nrHugePages * 2048)));
+ assertEquals(plugin.getVirtualMemorySize(),
+ 1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
+ }
+
@Test
public void testCoreCounts() throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org