You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ll...@apache.org on 2012/12/20 01:20:53 UTC
svn commit: r1424244 - in
/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn:
hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/
hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/
hadoop-yarn-server/hadoop-yarn-server-nodemanage...
Author: llu
Date: Thu Dec 20 00:20:53 2012
New Revision: 1424244
URL: http://svn.apache.org/viewvc?rev=1424244&view=rev
Log:
YARN-223. Update process tree instead of getting new process trees. (Radim Kolar via llu)
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java?rev=1424244&r1=1424243&r2=1424244&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java Thu Dec 20 00:20:53 2012
@@ -91,7 +91,6 @@ public class LinuxResourceCalculatorPlug
private float cpuUsage = UNAVAILABLE;
private long sampleTime = UNAVAILABLE;
private long lastSampleTime = UNAVAILABLE;
- private ResourceCalculatorProcessTree pTree = null;
boolean readMemInfoFile = false;
boolean readCpuInfoFile = false;
@@ -109,8 +108,6 @@ public class LinuxResourceCalculatorPlug
procfsCpuFile = PROCFS_CPUINFO;
procfsStatFile = PROCFS_STAT;
jiffyLengthInMillis = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS;
- String pid = System.getenv().get("JVM_PID");
- pTree = new ProcfsBasedProcessTree(pid);
}
/**
@@ -129,8 +126,6 @@ public class LinuxResourceCalculatorPlug
this.procfsCpuFile = procfsCpuFile;
this.procfsStatFile = procfsStatFile;
this.jiffyLengthInMillis = jiffyLengthInMillis;
- String pid = System.getenv().get("JVM_PID");
- pTree = new ProcfsBasedProcessTree(pid);
}
/**
@@ -400,13 +395,4 @@ public class LinuxResourceCalculatorPlug
}
System.out.println("CPU usage % : " + plugin.getCpuUsage());
}
-
- @Override
- public ProcResourceValues getProcResourceValues() {
- pTree = pTree.getProcessTree();
- long cpuTime = pTree.getCumulativeCpuTime();
- long pMem = pTree.getCumulativeRssmem();
- long vMem = pTree.getCumulativeVmem();
- return new ProcResourceValues(cpuTime, pMem, vMem);
- }
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1424244&r1=1424243&r2=1424244&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Thu Dec 20 00:20:53 2012
@@ -140,13 +140,12 @@ public class ProcfsBasedProcessTree exte
}
/**
- * Get the process-tree with latest state. If the root-process is not alive,
- * an empty tree will be returned.
+ * Update process-tree with latest state. If the root-process is not alive,
+ * tree will be empty.
*
- * @return the process-tree with latest state.
*/
@Override
- public ResourceCalculatorProcessTree getProcessTree() {
+ public void updateProcessTree() {
if (!pid.equals(deadPid)) {
// Get the list of processes
List<String> processList = getProcessList();
@@ -172,7 +171,7 @@ public class ProcfsBasedProcessTree exte
}
if (me == null) {
- return this;
+ return;
}
// Add each process to its parent.
@@ -214,7 +213,6 @@ public class ProcfsBasedProcessTree exte
LOG.debug(this.toString());
}
}
- return this;
}
/** Verify that the given process id is same as its process group id.
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java?rev=1424244&r1=1424243&r2=1424244&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java Thu Dec 20 00:20:53 2012
@@ -91,48 +91,6 @@ public abstract class ResourceCalculator
public abstract float getCpuUsage();
/**
- * Obtain resource status used by current process tree.
- */
- @InterfaceAudience.Private
- @InterfaceStability.Unstable
- public abstract ProcResourceValues getProcResourceValues();
-
- public static class ProcResourceValues {
- private final long cumulativeCpuTime;
- private final long physicalMemorySize;
- private final long virtualMemorySize;
- public ProcResourceValues(long cumulativeCpuTime, long physicalMemorySize,
- long virtualMemorySize) {
- this.cumulativeCpuTime = cumulativeCpuTime;
- this.physicalMemorySize = physicalMemorySize;
- this.virtualMemorySize = virtualMemorySize;
- }
- /**
- * Obtain the physical memory size used by current process tree.
- * @return physical memory size in bytes.
- */
- public long getPhysicalMemorySize() {
- return physicalMemorySize;
- }
-
- /**
- * Obtain the virtual memory size used by a current process tree.
- * @return virtual memory size in bytes.
- */
- public long getVirtualMemorySize() {
- return virtualMemorySize;
- }
-
- /**
- * Obtain the cumulative CPU time used by a current process tree.
- * @return cumulative CPU time in milliseconds
- */
- public long getCumulativeCpuTime() {
- return cumulativeCpuTime;
- }
- }
-
- /**
* Create the ResourceCalculatorPlugin from the class name and configure it. If
* class name is null, this method will try and return a memory calculator
* plugin available for this system.
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java?rev=1424244&r1=1424243&r2=1424244&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java Thu Dec 20 00:20:53 2012
@@ -43,16 +43,14 @@ public abstract class ResourceCalculator
}
/**
- * Get the process-tree with latest state. If the root-process is not alive,
- * an empty tree will be returned.
+ * Update the process-tree with latest state.
*
* Each call to this function should increment the age of the running
* processes that already exist in the process tree. Age is used other API's
* of the interface.
*
- * @return the process-tree with latest state.
*/
- public abstract ResourceCalculatorProcessTree getProcessTree();
+ public abstract void updateProcessTree();
/**
* Get a dump of the process-tree.
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java?rev=1424244&r1=1424243&r2=1424244&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java Thu Dec 20 00:20:53 2012
@@ -161,7 +161,7 @@ public class TestProcfsBasedProcessTree
String pid = getRogueTaskPID();
LOG.info("Root process pid: " + pid);
ProcfsBasedProcessTree p = createProcessTree(pid);
- p.getProcessTree(); // initialize
+ p.updateProcessTree(); // initialize
LOG.info("ProcessTree: " + p.toString());
File leaf = new File(lowestDescendant);
@@ -174,7 +174,7 @@ public class TestProcfsBasedProcessTree
}
}
- p.getProcessTree(); // reconstruct
+ p.updateProcessTree(); // reconstruct
LOG.info("ProcessTree: " + p.toString());
// Get the process-tree dump
@@ -213,7 +213,7 @@ public class TestProcfsBasedProcessTree
}
// ProcessTree is gone now. Any further calls should be sane.
- p.getProcessTree();
+ p.updateProcessTree();
Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
Assert.assertTrue("Cumulative vmem for the gone-process is "
+ p.getCumulativeVmem() + " . It should be zero.", p
@@ -358,7 +358,7 @@ public class TestProcfsBasedProcessTree
ProcfsBasedProcessTree processTree =
createProcessTree("100", procfsRootDir.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative memory
Assert.assertEquals("Cumulative virtual memory does not match", 600000L,
@@ -384,7 +384,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, pids, procInfos);
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative cpu time again
cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ?
@@ -431,7 +431,7 @@ public class TestProcfsBasedProcessTree
ProcfsBasedProcessTree processTree =
createProcessTree("100", procfsRootDir.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative memory
Assert.assertEquals("Cumulative memory does not match",
@@ -447,7 +447,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, newPids, newProcInfos);
// check memory includes the new process.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
Assert.assertEquals("Cumulative vmem does not include new process",
1200000L, processTree.getCumulativeVmem());
long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
@@ -473,7 +473,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, newPids, newProcInfos);
// refresh process tree
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// processes older than 2 iterations should be same as before.
Assert.assertEquals("Cumulative vmem shouldn't have included new processes",
@@ -577,7 +577,7 @@ public class TestProcfsBasedProcessTree
ProcfsBasedProcessTree processTree = createProcessTree(
"100", procfsRootDir.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// Get the process-tree dump
String processTreeDump = processTree.getProcessTreeDump();
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java?rev=1424244&r1=1424243&r2=1424244&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java Thu Dec 20 00:20:53 2012
@@ -34,8 +34,7 @@ public class TestResourceCalculatorProce
super(pid);
}
- public ResourceCalculatorProcessTree getProcessTree() {
- return this;
+ public void updateProcessTree() {
}
public String getProcessTreeDump() {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1424244&r1=1424243&r2=1424244&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Thu Dec 20 00:20:53 2012
@@ -396,9 +396,7 @@ public class ContainersMonitorImpl exten
LOG.debug("Constructing ProcessTree for : PID = " + pId
+ " ContainerId = " + containerId);
ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
- pTree = pTree.getProcessTree(); // get the updated process-tree
- ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
- // updated state
+ pTree.updateProcessTree(); // update process-tree
long currentVmemUsage = pTree.getCumulativeVmem();
long currentPmemUsage = pTree.getCumulativeRssmem();
// as processes begin with an age 1, we want to see if there
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1424244&r1=1424243&r2=1424244&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Thu Dec 20 00:20:53 2012
@@ -134,7 +134,7 @@ public class TestContainersMonitor exten
ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
"100",
procfsRootDir.getAbsolutePath());
- pTree.getProcessTree();
+ pTree.updateProcessTree();
assertTrue("tree rooted at 100 should be over limit " +
"after first iteration.",
test.isProcessTreeOverLimit(pTree, "dummyId", limit));
@@ -142,13 +142,13 @@ public class TestContainersMonitor exten
// the tree rooted at 200 is initially below limit.
pTree = new ProcfsBasedProcessTree("200",
procfsRootDir.getAbsolutePath());
- pTree.getProcessTree();
+ pTree.updateProcessTree();
assertFalse("tree rooted at 200 shouldn't be over limit " +
"after one iteration.",
test.isProcessTreeOverLimit(pTree, "dummyId", limit));
// second iteration - now the tree has been over limit twice,
// hence it should be declared over limit.
- pTree.getProcessTree();
+ pTree.updateProcessTree();
assertTrue(
"tree rooted at 200 should be over limit after 2 iterations",
test.isProcessTreeOverLimit(pTree, "dummyId", limit));
@@ -156,12 +156,12 @@ public class TestContainersMonitor exten
// the tree rooted at 600 is never over limit.
pTree = new ProcfsBasedProcessTree("600",
procfsRootDir.getAbsolutePath());
- pTree.getProcessTree();
+ pTree.updateProcessTree();
assertFalse("tree rooted at 600 should never be over limit.",
test.isProcessTreeOverLimit(pTree, "dummyId", limit));
// another iteration does not make any difference.
- pTree.getProcessTree();
+ pTree.updateProcessTree();
assertFalse("tree rooted at 600 should never be over limit.",
test.isProcessTreeOverLimit(pTree, "dummyId", limit));
} finally {