You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/03/30 21:37:23 UTC
[17/20] hadoop git commit: YARN-3304. Cleaning up
ResourceCalculatorProcessTree APIs for public use and removing
inconsistencies in the default values. Contributed by Junping Du and Karthik
Kambatla.
YARN-3304. Cleaning up ResourceCalculatorProcessTree APIs for public use and removing inconsistencies in the default values. Contributed by Junping Du and Karthik Kambatla.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e4f1b88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e4f1b88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e4f1b88
Branch: refs/heads/YARN-2928
Commit: 4e4f1b88dd29de11b9535194cc05f9db2a5570b1
Parents: 6baa8fd
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Mon Mar 30 10:09:40 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Mar 30 12:10:49 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapred/Task.java | 26 ++++--
hadoop-yarn-project/CHANGES.txt | 4 +
.../apache/hadoop/yarn/util/CpuTimeTracker.java | 3 +-
.../yarn/util/ProcfsBasedProcessTree.java | 80 +++++++++---------
.../util/ResourceCalculatorProcessTree.java | 66 ++++++++-------
.../yarn/util/WindowsBasedProcessTree.java | 21 +++--
.../yarn/util/TestProcfsBasedProcessTree.java | 85 ++++++++++----------
.../util/TestResourceCalculatorProcessTree.java | 4 +-
.../yarn/util/TestWindowsBasedProcessTree.java | 28 +++----
.../monitor/ContainerMetrics.java | 12 ++-
.../monitor/ContainersMonitorImpl.java | 12 +--
11 files changed, 187 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index bf5ca22..80881bc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -171,7 +171,7 @@ abstract public class Task implements Writable, Configurable {
skipRanges.skipRangeIterator();
private ResourceCalculatorProcessTree pTree;
- private long initCpuCumulativeTime = 0;
+ private long initCpuCumulativeTime = ResourceCalculatorProcessTree.UNAVAILABLE;
protected JobConf conf;
protected MapOutputFile mapOutputFile;
@@ -866,13 +866,25 @@ abstract public class Task implements Writable, Configurable {
}
pTree.updateProcessTree();
long cpuTime = pTree.getCumulativeCpuTime();
- long pMem = pTree.getCumulativeRssmem();
- long vMem = pTree.getCumulativeVmem();
+ long pMem = pTree.getRssMemorySize();
+ long vMem = pTree.getVirtualMemorySize();
// Remove the CPU time consumed previously by JVM reuse
- cpuTime -= initCpuCumulativeTime;
- counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
- counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
- counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
+ if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE &&
+ initCpuCumulativeTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ cpuTime -= initCpuCumulativeTime;
+ }
+
+ if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
+ }
+
+ if (pMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
+ }
+
+ if (vMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0d07032..3c16f24 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -897,6 +897,10 @@ Release 2.7.0 - UNRELEASED
YARN-2213. Change proxy-user cookie log in AmIpFilter to DEBUG.
(Varun Saxena via xgong)
+ YARN-3304. Cleaning up ResourceCalculatorProcessTree APIs for public use and
+ removing inconsistencies in the default values. (Junping Du and Karthik
+ Kambatla via vinodkv)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CpuTimeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CpuTimeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CpuTimeTracker.java
index d36848e..b09a4b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CpuTimeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CpuTimeTracker.java
@@ -26,7 +26,8 @@ import java.math.BigInteger;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class CpuTimeTracker {
- public static final int UNAVAILABLE = -1;
+ public static final int UNAVAILABLE =
+ ResourceCalculatorProcessTree.UNAVAILABLE;
final long MINIMUM_UPDATE_INTERVAL;
// CPU used time since system is on (ms)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
index 134cec2..9996a79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
@@ -140,7 +140,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
static private String deadPid = "-1";
private String pid = deadPid;
static private Pattern numberPattern = Pattern.compile("[1-9][0-9]*");
- private Long cpuTime = 0L;
+ private long cpuTime = UNAVAILABLE;
protected Map<String, ProcessInfo> processTree =
new HashMap<String, ProcessInfo>();
@@ -340,66 +340,53 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
return ret.toString();
}
- /**
- * Get the cumulative virtual memory used by all the processes in the
- * process-tree that are older than the passed in age.
- *
- * @param olderThanAge processes above this age are included in the
- * memory addition
- * @return cumulative virtual memory used by the process-tree in bytes,
- * for processes older than this age.
- */
@Override
- public long getCumulativeVmem(int olderThanAge) {
- long total = 0;
+ public long getVirtualMemorySize(int olderThanAge) {
+ long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.getAge() > olderThanAge)) {
+ if (total == UNAVAILABLE ) {
+ total = 0;
+ }
total += p.getVmem();
}
}
return total;
}
- /**
- * Get the cumulative resident set size (rss) memory used by all the processes
- * in the process-tree that are older than the passed in age.
- *
- * @param olderThanAge processes above this age are included in the
- * memory addition
- * @return cumulative rss memory used by the process-tree in bytes,
- * for processes older than this age. return 0 if it cannot be
- * calculated
- */
@Override
- public long getCumulativeRssmem(int olderThanAge) {
+ public long getRssMemorySize(int olderThanAge) {
if (PAGE_SIZE < 0) {
- return 0;
+ return UNAVAILABLE;
}
if (smapsEnabled) {
- return getSmapBasedCumulativeRssmem(olderThanAge);
+ return getSmapBasedRssMemorySize(olderThanAge);
}
+ boolean isAvailable = false;
long totalPages = 0;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.getAge() > olderThanAge)) {
totalPages += p.getRssmemPage();
+ isAvailable = true;
}
}
- return totalPages * PAGE_SIZE; // convert # pages to byte
+ return isAvailable ? totalPages * PAGE_SIZE : UNAVAILABLE; // convert # pages to byte
}
/**
- * Get the cumulative resident set size (RSS) memory used by all the processes
+ * Get the resident set size (RSS) memory used by all the processes
* in the process-tree that are older than the passed in age. RSS is
* calculated based on SMAP information. Skip mappings with "r--s", "r-xs"
* permissions to get real RSS usage of the process.
*
* @param olderThanAge
* processes above this age are included in the memory addition
- * @return cumulative rss memory used by the process-tree in bytes, for
- * processes older than this age. return 0 if it cannot be calculated
+ * @return rss memory used by the process-tree in bytes, for
+ * processes older than this age. return {@link #UNAVAILABLE} if it cannot
+ * be calculated.
*/
- private long getSmapBasedCumulativeRssmem(int olderThanAge) {
- long total = 0;
+ private long getSmapBasedRssMemorySize(int olderThanAge) {
+ long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.getAge() > olderThanAge)) {
ProcessTreeSmapMemInfo procMemInfo = processSMAPTree.get(p.getPid());
@@ -412,6 +399,9 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
.equalsIgnoreCase(READ_EXECUTE_WITH_SHARED_PERMISSION)) {
continue;
}
+ if (total == UNAVAILABLE){
+ total = 0;
+ }
total +=
Math.min(info.sharedDirty, info.pss) + info.privateDirty
+ info.privateClean;
@@ -429,30 +419,34 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
}
}
}
- total = (total * KB_TO_BYTES); // convert to bytes
+ if (total > 0) {
+ total *= KB_TO_BYTES; // convert to bytes
+ }
LOG.info("SmapBasedCumulativeRssmem (bytes) : " + total);
return total; // size
}
- /**
- * Get the CPU time in millisecond used by all the processes in the
- * process-tree since the process-tree created
- *
- * @return cumulative CPU time in millisecond since the process-tree created
- * return 0 if it cannot be calculated
- */
@Override
public long getCumulativeCpuTime() {
if (JIFFY_LENGTH_IN_MILLIS < 0) {
- return 0;
+ return UNAVAILABLE;
}
long incJiffies = 0;
+ boolean isAvailable = false;
for (ProcessInfo p : processTree.values()) {
if (p != null) {
incJiffies += p.getDtime();
+ // data is available
+ isAvailable = true;
+ }
+ }
+ if (isAvailable) {
+ // reset cpuTime to 0 instead of UNAVAILABLE
+ if (cpuTime == UNAVAILABLE) {
+ cpuTime = 0L;
}
+ cpuTime += incJiffies * JIFFY_LENGTH_IN_MILLIS;
}
- cpuTime += incJiffies * JIFFY_LENGTH_IN_MILLIS;
return cpuTime;
}
@@ -1031,8 +1025,8 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
System.out.println("Cpu usage " + procfsBasedProcessTree
.getCpuUsagePercent());
System.out.println("Vmem usage in bytes " + procfsBasedProcessTree
- .getCumulativeVmem());
+ .getVirtualMemorySize());
System.out.println("Rss mem usage in bytes " + procfsBasedProcessTree
- .getCumulativeRssmem());
+ .getRssMemorySize());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
index 6ee8834..3c4bf52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
@@ -23,19 +23,23 @@ import java.lang.reflect.Constructor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
/**
* Interface class to obtain process resource usage
- *
+ * NOTE: This class should not be used by external users, but only by external
+ * developers to extend and include their own process-tree implementation,
+ * especially for platforms other than Linux and Windows.
*/
@Public
@Evolving
public abstract class ResourceCalculatorProcessTree extends Configured {
static final Log LOG = LogFactory
.getLog(ResourceCalculatorProcessTree.class);
+ public static final int UNAVAILABLE = -1;
/**
* Create process-tree instance with specified root process.
@@ -65,63 +69,64 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
public abstract String getProcessTreeDump();
/**
- * Get the cumulative virtual memory used by all the processes in the
+ * Get the virtual memory used by all the processes in the
* process-tree.
*
- * @return cumulative virtual memory used by the process-tree in bytes.
+ * @return virtual memory used by the process-tree in bytes,
+ * {@link #UNAVAILABLE} if it cannot be calculated.
*/
- public long getCumulativeVmem() {
- return getCumulativeVmem(0);
+ public long getVirtualMemorySize() {
+ return getVirtualMemorySize(0);
}
/**
- * Get the cumulative resident set size (rss) memory used by all the processes
+ * Get the resident set size (rss) memory used by all the processes
* in the process-tree.
*
- * @return cumulative rss memory used by the process-tree in bytes. return 0
- * if it cannot be calculated
+ * @return rss memory used by the process-tree in bytes,
+ * {@link #UNAVAILABLE} if it cannot be calculated.
*/
- public long getCumulativeRssmem() {
- return getCumulativeRssmem(0);
+ public long getRssMemorySize() {
+ return getRssMemorySize(0);
}
/**
- * Get the cumulative virtual memory used by all the processes in the
+ * Get the virtual memory used by all the processes in the
* process-tree that are older than the passed in age.
*
* @param olderThanAge processes above this age are included in the
- * memory addition
- * @return cumulative virtual memory used by the process-tree in bytes,
- * for processes older than this age. return 0 if it cannot be
- * calculated
+ * memory addition
+ * @return virtual memory used by the process-tree in bytes for
+ * processes older than the specified age, {@link #UNAVAILABLE} if it
+ * cannot be calculated.
*/
- public long getCumulativeVmem(int olderThanAge) {
- return 0;
+ public long getVirtualMemorySize(int olderThanAge) {
+ return UNAVAILABLE;
}
/**
- * Get the cumulative resident set size (rss) memory used by all the processes
+ * Get the resident set size (rss) memory used by all the processes
* in the process-tree that are older than the passed in age.
*
* @param olderThanAge processes above this age are included in the
- * memory addition
- * @return cumulative rss memory used by the process-tree in bytes,
- * for processes older than this age. return 0 if it cannot be
- * calculated
+ * memory addition
+ * @return rss memory used by the process-tree in bytes for
+ * processes older than specified age, {@link #UNAVAILABLE} if it cannot be
+ * calculated.
*/
- public long getCumulativeRssmem(int olderThanAge) {
- return 0;
+ public long getRssMemorySize(int olderThanAge) {
+ return UNAVAILABLE;
}
/**
* Get the CPU time in millisecond used by all the processes in the
* process-tree since the process-tree was created
*
- * @return cumulative CPU time in millisecond since the process-tree created
- * return 0 if it cannot be calculated
+ * @return cumulative CPU time in millisecond since the process-tree
+ * created, {@link #UNAVAILABLE} if it cannot be calculated.
*/
public long getCumulativeCpuTime() {
- return 0;
+ return UNAVAILABLE;
}
/**
@@ -129,11 +134,11 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
* average between samples as a ratio of overall CPU cycles similar to top.
* Thus, if 2 out of 4 cores are used this should return 200.0.
*
- * @return percentage CPU usage since the process-tree was created
- * return {@link CpuTimeTracker#UNAVAILABLE} if it cannot be calculated
+ * @return percentage CPU usage since the process-tree was created,
+ * {@link #UNAVAILABLE} if it cannot be calculated.
*/
public float getCpuUsagePercent() {
- return -1;
+ return UNAVAILABLE;
}
/** Verify that the tree process id is same as its process group id.
@@ -153,6 +158,7 @@ public abstract class ResourceCalculatorProcessTree extends Configured {
* @return ResourceCalculatorProcessTree or null if ResourceCalculatorPluginTree
* is not available for this system.
*/
+ @Private
public static ResourceCalculatorProcessTree getResourceCalculatorProcessTree(
String pid, Class<? extends ResourceCalculatorProcessTree> clazz, Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
index 5c3251f..90426be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
@@ -45,8 +45,8 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
}
private String taskProcessId = null;
- private long cpuTimeMs = 0;
- private Map<String, ProcessInfo> processTree =
+ private long cpuTimeMs = UNAVAILABLE;
+ private Map<String, ProcessInfo> processTree =
new HashMap<String, ProcessInfo>();
public static boolean isAvailable() {
@@ -173,10 +173,13 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
}
@Override
- public long getCumulativeVmem(int olderThanAge) {
- long total = 0;
+ public long getVirtualMemorySize(int olderThanAge) {
+ long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.age > olderThanAge)) {
+ if (total == UNAVAILABLE) {
+ total = 0;
+ }
total += p.vmem;
}
}
@@ -184,10 +187,13 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
}
@Override
- public long getCumulativeRssmem(int olderThanAge) {
- long total = 0;
+ public long getRssMemorySize(int olderThanAge) {
+ long total = UNAVAILABLE;
for (ProcessInfo p : processTree.values()) {
if ((p != null) && (p.age > olderThanAge)) {
+ if (total == UNAVAILABLE) {
+ total = 0;
+ }
total += p.workingSet;
}
}
@@ -197,6 +203,9 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
@Override
public long getCumulativeCpuTime() {
for (ProcessInfo p : processTree.values()) {
+ if (cpuTimeMs == UNAVAILABLE) {
+ cpuTimeMs = 0;
+ }
cpuTimeMs += p.cpuTimeMsDelta;
}
return cpuTimeMs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
index d62e21d..eeeeb52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.util;
import static org.apache.hadoop.yarn.util.ProcfsBasedProcessTree.KB_TO_BYTES;
+import static org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.UNAVAILABLE;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;
@@ -226,8 +227,8 @@ public class TestProcfsBasedProcessTree {
p.updateProcessTree();
Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
Assert.assertTrue(
- "Cumulative vmem for the gone-process is " + p.getCumulativeVmem()
- + " . It should be zero.", p.getCumulativeVmem() == 0);
+ "vmem for the gone-process is " + p.getVirtualMemorySize()
+ + " . It should be zero.", p.getVirtualMemorySize() == 0);
Assert.assertTrue(p.toString().equals("[ ]"));
}
@@ -429,16 +430,16 @@ public class TestProcfsBasedProcessTree {
// build the process tree.
processTree.updateProcessTree();
- // verify cumulative memory
- Assert.assertEquals("Cumulative virtual memory does not match", 600000L,
- processTree.getCumulativeVmem());
+ // verify virtual memory
+ Assert.assertEquals("Virtual memory does not match", 600000L,
+ processTree.getVirtualMemorySize());
// verify rss memory
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 600L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
- Assert.assertEquals("Cumulative rss memory does not match", cumuRssMem,
- processTree.getCumulativeRssmem());
+ Assert.assertEquals("rss memory does not match", cumuRssMem,
+ processTree.getRssMemorySize());
// verify cumulative cpu time
long cumuCpuTime =
@@ -456,8 +457,8 @@ public class TestProcfsBasedProcessTree {
setSmapsInProceTree(processTree, true);
// RSS=Min(shared_dirty,PSS)+PrivateClean+PrivateDirty (exclude r-xs,
// r--s)
- Assert.assertEquals("Cumulative rss memory does not match",
- (100 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem());
+ Assert.assertEquals("rss memory does not match",
+ (100 * KB_TO_BYTES * 3), processTree.getRssMemorySize());
// test the cpu time again to see if it cumulates
procInfos[0] =
@@ -563,9 +564,9 @@ public class TestProcfsBasedProcessTree {
new SystemClock());
setSmapsInProceTree(processTree, smapEnabled);
- // verify cumulative memory
+ // verify virtual memory
Assert.assertEquals("Cumulative memory does not match", 700000L,
- processTree.getCumulativeVmem());
+ processTree.getVirtualMemorySize());
// write one more process as child of 100.
String[] newPids = { "500" };
setupPidDirs(procfsRootDir, newPids);
@@ -581,34 +582,34 @@ public class TestProcfsBasedProcessTree {
// check memory includes the new process.
processTree.updateProcessTree();
- Assert.assertEquals("Cumulative vmem does not include new process",
- 1200000L, processTree.getCumulativeVmem());
+ Assert.assertEquals("vmem does not include new process",
+ 1200000L, processTree.getVirtualMemorySize());
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
- Assert.assertEquals("Cumulative rssmem does not include new process",
- cumuRssMem, processTree.getCumulativeRssmem());
+ Assert.assertEquals("rssmem does not include new process",
+ cumuRssMem, processTree.getRssMemorySize());
} else {
- Assert.assertEquals("Cumulative rssmem does not include new process",
- 100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem());
+ Assert.assertEquals("rssmem does not include new process",
+ 100 * KB_TO_BYTES * 4, processTree.getRssMemorySize());
}
// however processes older than 1 iteration will retain the older value
Assert.assertEquals(
- "Cumulative vmem shouldn't have included new process", 700000L,
- processTree.getCumulativeVmem(1));
+ "vmem shouldn't have included new process", 700000L,
+ processTree.getVirtualMemorySize(1));
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
Assert.assertEquals(
- "Cumulative rssmem shouldn't have included new process", cumuRssMem,
- processTree.getCumulativeRssmem(1));
+ "rssmem shouldn't have included new process", cumuRssMem,
+ processTree.getRssMemorySize(1));
} else {
Assert.assertEquals(
- "Cumulative rssmem shouldn't have included new process",
- 100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1));
+ "rssmem shouldn't have included new process",
+ 100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1));
}
// one more process
@@ -629,49 +630,49 @@ public class TestProcfsBasedProcessTree {
// processes older than 2 iterations should be same as before.
Assert.assertEquals(
- "Cumulative vmem shouldn't have included new processes", 700000L,
- processTree.getCumulativeVmem(2));
+ "vmem shouldn't have included new processes", 700000L,
+ processTree.getVirtualMemorySize(2));
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
Assert.assertEquals(
- "Cumulative rssmem shouldn't have included new processes",
- cumuRssMem, processTree.getCumulativeRssmem(2));
+ "rssmem shouldn't have included new processes",
+ cumuRssMem, processTree.getRssMemorySize(2));
} else {
Assert.assertEquals(
- "Cumulative rssmem shouldn't have included new processes",
- 100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2));
+ "rssmem shouldn't have included new processes",
+ 100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2));
}
// processes older than 1 iteration should not include new process,
// but include process 500
Assert.assertEquals(
- "Cumulative vmem shouldn't have included new processes", 1200000L,
- processTree.getCumulativeVmem(1));
+ "vmem shouldn't have included new processes", 1200000L,
+ processTree.getVirtualMemorySize(1));
if (!smapEnabled) {
long cumuRssMem =
ProcfsBasedProcessTree.PAGE_SIZE > 0
? 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
Assert.assertEquals(
- "Cumulative rssmem shouldn't have included new processes",
- cumuRssMem, processTree.getCumulativeRssmem(1));
+ "rssmem shouldn't have included new processes",
+ cumuRssMem, processTree.getRssMemorySize(1));
} else {
Assert.assertEquals(
- "Cumulative rssmem shouldn't have included new processes",
- 100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1));
+ "rssmem shouldn't have included new processes",
+ 100 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1));
}
- // no processes older than 3 iterations, this should be 0
+ // no processes older than 3 iterations
Assert.assertEquals(
- "Getting non-zero vmem for processes older than 3 iterations", 0L,
- processTree.getCumulativeVmem(3));
+ "Getting non-zero vmem for processes older than 3 iterations",
+ UNAVAILABLE, processTree.getVirtualMemorySize(3));
Assert.assertEquals(
- "Getting non-zero rssmem for processes older than 3 iterations", 0L,
- processTree.getCumulativeRssmem(3));
+ "Getting non-zero rssmem for processes older than 3 iterations",
+ UNAVAILABLE, processTree.getRssMemorySize(3));
Assert.assertEquals(
- "Getting non-zero rssmem for processes older than 3 iterations", 0L,
- processTree.getCumulativeRssmem(3));
+ "Getting non-zero rssmem for processes older than 3 iterations",
+ UNAVAILABLE, processTree.getRssMemorySize(3));
} finally {
FileUtil.fullyDelete(procfsRootDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
index eaf7e8e..9bf525c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
@@ -41,11 +41,11 @@ public class TestResourceCalculatorProcessTree {
return "Empty tree for testing";
}
- public long getCumulativeRssmem(int age) {
+ public long getRssMemorySize(int age) {
return 0;
}
- public long getCumulativeVmem(int age) {
+ public long getVirtualMemorySize(int age) {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
index d5b5c37..2a208a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
@@ -53,26 +53,26 @@ public class TestWindowsBasedProcessTree {
WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1");
pTree.infoStr = "3524,1024,1024,500\r\n2844,1024,1024,500\r\n";
pTree.updateProcessTree();
- assertTrue(pTree.getCumulativeVmem() == 2048);
- assertTrue(pTree.getCumulativeVmem(0) == 2048);
- assertTrue(pTree.getCumulativeRssmem() == 2048);
- assertTrue(pTree.getCumulativeRssmem(0) == 2048);
+ assertTrue(pTree.getVirtualMemorySize() == 2048);
+ assertTrue(pTree.getVirtualMemorySize(0) == 2048);
+ assertTrue(pTree.getRssMemorySize() == 2048);
+ assertTrue(pTree.getRssMemorySize(0) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 1000);
pTree.infoStr = "3524,1024,1024,1000\r\n2844,1024,1024,1000\r\n1234,1024,1024,1000\r\n";
pTree.updateProcessTree();
- assertTrue(pTree.getCumulativeVmem() == 3072);
- assertTrue(pTree.getCumulativeVmem(1) == 2048);
- assertTrue(pTree.getCumulativeRssmem() == 3072);
- assertTrue(pTree.getCumulativeRssmem(1) == 2048);
- assertTrue(pTree.getCumulativeCpuTime() == 3000);
+ assertTrue(pTree.getVirtualMemorySize() == 3072);
+ assertTrue(pTree.getVirtualMemorySize(1) == 2048);
+ assertTrue(pTree.getRssMemorySize() == 3072);
+ assertTrue(pTree.getRssMemorySize(1) == 2048);
+ assertTrue(pTree.getCumulativeCpuTime() == 3000);
pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n";
pTree.updateProcessTree();
- assertTrue(pTree.getCumulativeVmem() == 2048);
- assertTrue(pTree.getCumulativeVmem(2) == 2048);
- assertTrue(pTree.getCumulativeRssmem() == 2048);
- assertTrue(pTree.getCumulativeRssmem(2) == 2048);
- assertTrue(pTree.getCumulativeCpuTime() == 4000);
+ assertTrue(pTree.getVirtualMemorySize() == 2048);
+ assertTrue(pTree.getVirtualMemorySize(2) == 2048);
+ assertTrue(pTree.getRssMemorySize() == 2048);
+ assertTrue(pTree.getRssMemorySize(2) == 2048);
+ assertTrue(pTree.getCumulativeCpuTime() == 4000);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
index 1375da8..ffa72a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
@@ -188,13 +188,19 @@ public class ContainerMetrics implements MetricsSource {
}
public void recordMemoryUsage(int memoryMBs) {
- this.pMemMBsStat.add(memoryMBs);
+ if (memoryMBs >= 0) {
+ this.pMemMBsStat.add(memoryMBs);
+ }
}
public void recordCpuUsage(
int totalPhysicalCpuPercent, int milliVcoresUsed) {
- this.cpuCoreUsagePercent.add(totalPhysicalCpuPercent);
- this.milliVcoresUsed.add(milliVcoresUsed);
+ if (totalPhysicalCpuPercent >=0) {
+ this.cpuCoreUsagePercent.add(totalPhysicalCpuPercent);
+ }
+ if (milliVcoresUsed >= 0) {
+ this.milliVcoresUsed.add(milliVcoresUsed);
+ }
}
public void recordProcessId(String processId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e4f1b88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index b587e46..5153051 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -333,10 +333,10 @@ public class ContainersMonitorImpl extends AbstractService implements
// method provided just for easy testing purposes
boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree,
String containerId, long limit) {
- long currentMemUsage = pTree.getCumulativeVmem();
+ long currentMemUsage = pTree.getVirtualMemorySize();
// as processes begin with an age 1, we want to see if there are processes
// more than 1 iteration old.
- long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
+ long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
return isProcessTreeOverLimit(containerId, currentMemUsage,
curMemUsageOfAgedProcesses, limit);
}
@@ -437,8 +437,8 @@ public class ContainersMonitorImpl extends AbstractService implements
+ " ContainerId = " + containerId);
ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
pTree.updateProcessTree(); // update process-tree
- long currentVmemUsage = pTree.getCumulativeVmem();
- long currentPmemUsage = pTree.getCumulativeRssmem();
+ long currentVmemUsage = pTree.getVirtualMemorySize();
+ long currentPmemUsage = pTree.getRssMemorySize();
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300% and
// cpuUsageTotalCoresPercentage should be 50%
@@ -451,8 +451,8 @@ public class ContainersMonitorImpl extends AbstractService implements
* maxVCoresAllottedForContainers /nodeCpuPercentageForYARN);
// as processes begin with an age 1, we want to see if there
// are processes more than 1 iteration old.
- long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
- long curRssMemUsageOfAgedProcesses = pTree.getCumulativeRssmem(1);
+ long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
+ long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1);
long vmemLimit = ptInfo.getVmemLimit();
long pmemLimit = ptInfo.getPmemLimit();
LOG.info(String.format(