You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2015/12/22 02:06:57 UTC
[2/3] hadoop git commit: YARN-3458. CPU resource monitoring in
Windows. Contributed by Inigo Goiri.
YARN-3458. CPU resource monitoring in Windows. Contributed by Inigo Goiri.
(cherry picked from commit 114b59095540bb80db5153c816f9d285e4029031)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdfcffb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdfcffb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdfcffb3
Branch: refs/heads/branch-2
Commit: fdfcffb3d1054ae9587a66e4919a0bb10720df55
Parents: daf5fae
Author: cnauroth <cn...@apache.org>
Authored: Mon Dec 21 16:59:09 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Mon Dec 21 16:59:17 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../yarn/util/WindowsBasedProcessTree.java | 51 ++++++++++++++++++--
.../yarn/util/TestWindowsBasedProcessTree.java | 25 ++++++++--
3 files changed, 68 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdfcffb3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b0b3738..93b07c2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -228,6 +228,8 @@ Release 2.8.0 - UNRELEASED
YARN-3226. UI changes for decommissioning node. (Sunil G via
junping_du)
+ YARN-3458. CPU resource monitoring in Windows. (Inigo Goiri via cnauroth)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdfcffb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
index 1bf25a5..30add01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.yarn.util;
import java.io.IOException;
+import java.math.BigInteger;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.util.CpuTimeTracker;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.util.StringUtils;
@@ -48,7 +50,12 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
private long cpuTimeMs = UNAVAILABLE;
private Map<String, ProcessInfo> processTree =
new HashMap<String, ProcessInfo>();
-
+
+ /** Track CPU utilization. */
+ private final CpuTimeTracker cpuTimeTracker;
+ /** Clock to account for CPU utilization. */
+ private Clock clock;
+
public static boolean isAvailable() {
if (Shell.WINDOWS) {
if (!Shell.hasWinutilsPath()) {
@@ -71,9 +78,25 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
return false;
}
- public WindowsBasedProcessTree(String pid) {
+ /**
+ * Create a monitor for a Windows process tree.
+ * @param pid Identifier of the job object.
+ */
+ public WindowsBasedProcessTree(final String pid) {
+ this(pid, new SystemClock());
+ }
+
+ /**
+ * Create a monitor for a Windows process tree.
+ * @param pid Identifier of the job object.
+ * @param pClock Clock to keep track of time for CPU utilization.
+ */
+ public WindowsBasedProcessTree(final String pid, final Clock pClock) {
super(pid);
- taskProcessId = pid;
+ this.taskProcessId = pid;
+ this.clock = pClock;
+ // Instead of jiffies, Windows uses milliseconds directly; 1ms = 1 jiffy
+ this.cpuTimeTracker = new CpuTimeTracker(1L);
}
// helper method to override while testing
@@ -213,7 +236,7 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
}
return total;
}
-
+
@Override
@SuppressWarnings("deprecation")
public long getCumulativeRssmem(int olderThanAge) {
@@ -231,9 +254,27 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
return cpuTimeMs;
}
+ /**
+ * Get the number of used ms for all the processes under the monitored job
+ * object.
+ * @return Total consumed milliseconds by all processes in the job object.
+ */
+ private BigInteger getTotalProcessMs() {
+ long totalMs = 0;
+ for (ProcessInfo p : processTree.values()) {
+ if (p != null) {
+ totalMs += p.cpuTimeMs;
+ }
+ }
+ return BigInteger.valueOf(totalMs);
+ }
+
@Override
public float getCpuUsagePercent() {
- return UNAVAILABLE;
+ BigInteger processTotalMs = getTotalProcessMs();
+ cpuTimeTracker.updateElapsedJiffies(processTotalMs, clock.getTime());
+
+ return cpuTimeTracker.getCpuTrackerUsagePercent();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdfcffb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
index 80c5b02..c28915c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java
@@ -21,8 +21,9 @@ package org.apache.hadoop.yarn.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.Shell;
-
+import org.junit.Assert;
import org.junit.Test;
+
import static org.junit.Assert.assertTrue;
public class TestWindowsBasedProcessTree {
@@ -31,8 +32,9 @@ public class TestWindowsBasedProcessTree {
class WindowsBasedProcessTreeTester extends WindowsBasedProcessTree {
String infoStr = null;
- public WindowsBasedProcessTreeTester(String pid) {
- super(pid);
+
+ public WindowsBasedProcessTreeTester(String pid, Clock clock) {
+ super(pid, clock);
}
@Override
String getAllProcessInfoFromShell() {
@@ -49,8 +51,11 @@ public class TestWindowsBasedProcessTree {
}
assertTrue("WindowsBasedProcessTree should be available on Windows",
WindowsBasedProcessTree.isAvailable());
-
- WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1");
+ ControlledClock testClock = new ControlledClock(new SystemClock());
+ long elapsedTimeBetweenUpdatesMsec = 0;
+ testClock.setTime(elapsedTimeBetweenUpdatesMsec);
+
+ WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1", testClock);
pTree.infoStr = "3524,1024,1024,500\r\n2844,1024,1024,500\r\n";
pTree.updateProcessTree();
assertTrue(pTree.getVirtualMemorySize() == 2048);
@@ -63,8 +68,11 @@ public class TestWindowsBasedProcessTree {
assertTrue(pTree.getRssMemorySize(0) == 2048);
assertTrue(pTree.getCumulativeRssmem(0) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 1000);
+ assertTrue(pTree.getCpuUsagePercent() == ResourceCalculatorProcessTree.UNAVAILABLE);
pTree.infoStr = "3524,1024,1024,1000\r\n2844,1024,1024,1000\r\n1234,1024,1024,1000\r\n";
+ elapsedTimeBetweenUpdatesMsec = 1000;
+ testClock.setTime(elapsedTimeBetweenUpdatesMsec);
pTree.updateProcessTree();
assertTrue(pTree.getVirtualMemorySize() == 3072);
assertTrue(pTree.getCumulativeVmem() == 3072);
@@ -75,8 +83,13 @@ public class TestWindowsBasedProcessTree {
assertTrue(pTree.getRssMemorySize(1) == 2048);
assertTrue(pTree.getCumulativeRssmem(1) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 3000);
+ assertTrue(pTree.getCpuUsagePercent() == 200);
+ Assert.assertEquals("Percent CPU time is not correct",
+ pTree.getCpuUsagePercent(), 200, 0.01);
pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n";
+ elapsedTimeBetweenUpdatesMsec = 2000;
+ testClock.setTime(elapsedTimeBetweenUpdatesMsec);
pTree.updateProcessTree();
assertTrue(pTree.getVirtualMemorySize() == 2048);
assertTrue(pTree.getCumulativeVmem() == 2048);
@@ -87,5 +100,7 @@ public class TestWindowsBasedProcessTree {
assertTrue(pTree.getRssMemorySize(2) == 2048);
assertTrue(pTree.getCumulativeRssmem(2) == 2048);
assertTrue(pTree.getCumulativeCpuTime() == 4000);
+ Assert.assertEquals("Percent CPU time is not correct",
+ pTree.getCpuUsagePercent(), 0, 0.01);
}
}