You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/05/12 22:49:43 UTC
[26/36] hadoop git commit: YARN-3513. Remove unused variables in
ContainersMonitorImpl and add debug log for overall resource usage by all
containers. Contributed by Naganarasimha G R.
YARN-3513. Remove unused variables in ContainersMonitorImpl and add debug
log for overall resource usage by all containers. Contributed by
Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65935b00
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65935b00
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65935b00
Branch: refs/heads/YARN-2928
Commit: 65935b00437c12a80d0f0bb1c7c32bc6f7123588
Parents: abf9e44
Author: Devaraj K <de...@apache.org>
Authored: Tue May 12 16:54:38 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue May 12 13:44:26 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../monitor/ContainersMonitorImpl.java | 27 ++++++++++++++------
2 files changed, 22 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/65935b00/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 18a61e7..78bda68 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -300,6 +300,9 @@ Release 2.8.0 - UNRELEASED
YARN-3587. Fix the javadoc of DelegationTokenSecretManager in yarn, etc.
projects. (Gabor Liptak via junping_du)
+ YARN-3513. Remove unused variables in ContainersMonitorImpl and add debug
+ log for overall resource usage by all containers. (Naganarasimha G R via devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/65935b00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index ebd43d6..3db999b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -445,8 +445,10 @@ public class ContainersMonitorImpl extends AbstractService implements
// Now do the monitoring for the trackingContainers
// Check memory usage and kill any overflowing containers
- long vmemStillInUsage = 0;
- long pmemStillInUsage = 0;
+ long vmemUsageByAllContainers = 0;
+ long pmemByAllContainers = 0;
+ long cpuUsagePercentPerCoreByAllContainers = 0;
+ long cpuUsageTotalCoresByAllContainers = 0;
for (Iterator<Map.Entry<ContainerId, ProcessTreeInfo>> it =
trackingContainers.entrySet().iterator(); it.hasNext();) {
@@ -587,6 +589,13 @@ public class ContainersMonitorImpl extends AbstractService implements
containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
}
+ // Accounting the total memory in usage for all containers
+ vmemUsageByAllContainers += currentVmemUsage;
+ pmemByAllContainers += currentPmemUsage;
+ // Accounting the total cpu usage for all containers
+ cpuUsagePercentPerCoreByAllContainers += cpuUsagePercentPerCore;
+ cpuUsageTotalCoresByAllContainers += cpuUsagePercentPerCore;
+
if (isMemoryOverLimit) {
// Virtual or physical memory over limit. Fail the container and
// remove
@@ -603,12 +612,6 @@ public class ContainersMonitorImpl extends AbstractService implements
containerExitStatus, msg));
it.remove();
LOG.info("Removed ProcessTree with root " + pId);
- } else {
- // Accounting the total memory in usage for all containers that
- // are still
- // alive and within limits.
- vmemStillInUsage += currentVmemUsage;
- pmemStillInUsage += currentPmemUsage;
}
} catch (Exception e) {
@@ -629,6 +632,14 @@ public class ContainersMonitorImpl extends AbstractService implements
}
}
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Total Resource Usage stats in NM by all containers : "
+ + "Virtual Memory= " + vmemUsageByAllContainers
+ + ", Physical Memory= " + pmemByAllContainers
+ + ", Total CPU usage= " + cpuUsageTotalCoresByAllContainers
+ + ", Total CPU(% per core) usage"
+ + cpuUsagePercentPerCoreByAllContainers);
+ }
try {
Thread.sleep(monitoringInterval);