You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by su...@apache.org on 2013/01/03 04:42:15 UTC
svn commit: r1428155 - in
/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client:
hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapre...
Author: suresh
Date: Thu Jan 3 03:42:15 2013
New Revision: 1428155
URL: http://svn.apache.org/viewvc?rev=1428155&view=rev
Log:
YARN-223. Change processTree interface to work better with native code. Merge patch from trunk.
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?rev=1428155&r1=1428154&r2=1428155&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java Thu Jan 3 03:42:15 2013
@@ -61,8 +61,7 @@ import org.apache.hadoop.mapreduce.MRCon
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
-import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
-import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin.*;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
@@ -169,7 +168,7 @@ abstract public class Task implements Wr
private Iterator<Long> currentRecIndexIterator =
skipRanges.skipRangeIterator();
- private ResourceCalculatorPlugin resourceCalculator = null;
+ private ResourceCalculatorProcessTree pTree;
private long initCpuCumulativeTime = 0;
protected JobConf conf;
@@ -372,7 +371,7 @@ abstract public class Task implements Wr
* Return current state of the task.
* needs to be synchronized as communication thread
* sends the state every second
- * @return
+ * @return task state
*/
synchronized TaskStatus.State getState(){
return this.taskStatus.getRunState();
@@ -558,15 +557,15 @@ abstract public class Task implements Wr
}
}
committer.setupTask(taskContext);
- Class<? extends ResourceCalculatorPlugin> clazz =
- conf.getClass(MRConfig.RESOURCE_CALCULATOR_PLUGIN,
- null, ResourceCalculatorPlugin.class);
- resourceCalculator = ResourceCalculatorPlugin
- .getResourceCalculatorPlugin(clazz, conf);
- LOG.info(" Using ResourceCalculatorPlugin : " + resourceCalculator);
- if (resourceCalculator != null) {
- initCpuCumulativeTime =
- resourceCalculator.getProcResourceValues().getCumulativeCpuTime();
+ Class<? extends ResourceCalculatorProcessTree> clazz =
+ conf.getClass(MRConfig.RESOURCE_CALCULATOR_PROCESS_TREE,
+ null, ResourceCalculatorProcessTree.class);
+ pTree = ResourceCalculatorProcessTree
+ .getResourceCalculatorProcessTree(System.getenv().get("JVM_PID"), clazz, conf);
+ LOG.info(" Using ResourceCalculatorProcessTree : " + pTree);
+ if (pTree != null) {
+ pTree.updateProcessTree();
+ initCpuCumulativeTime = pTree.getCumulativeCpuTime();
}
}
@@ -817,14 +816,14 @@ abstract public class Task implements Wr
// Update generic resource counters
updateHeapUsageCounter();
- // Updating resources specified in ResourceCalculatorPlugin
- if (resourceCalculator == null) {
+ // Updating resources specified in ResourceCalculatorProcessTree
+ if (pTree == null) {
return;
}
- ProcResourceValues res = resourceCalculator.getProcResourceValues();
- long cpuTime = res.getCumulativeCpuTime();
- long pMem = res.getPhysicalMemorySize();
- long vMem = res.getVirtualMemorySize();
+ pTree.updateProcessTree();
+ long cpuTime = pTree.getCumulativeCpuTime();
+ long pMem = pTree.getCumulativeRssmem();
+ long vMem = pTree.getCumulativeVmem();
// Remove the CPU time consumed previously by JVM reuse
cpuTime -= initCpuCumulativeTime;
counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java?rev=1428155&r1=1428154&r2=1428155&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java Thu Jan 3 03:42:15 2013
@@ -55,8 +55,8 @@ public interface MRConfig {
public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
7*24*60*60*1000; // 7 days
- public static final String RESOURCE_CALCULATOR_PLUGIN =
- "mapreduce.job.resourcecalculatorplugin";
+ public static final String RESOURCE_CALCULATOR_PROCESS_TREE =
+ "mapreduce.job.process-tree.class";
public static final String STATIC_RESOLUTIONS =
"mapreduce.job.net.static.resolutions";
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java?rev=1428155&r1=1428154&r2=1428155&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java Thu Jan 3 03:42:15 2013
@@ -409,7 +409,7 @@ public class LinuxResourceCalculatorPlug
@Override
public ProcResourceValues getProcResourceValues() {
- pTree = pTree.getProcessTree();
+ pTree.updateProcessTree();
long cpuTime = pTree.getCumulativeCpuTime();
long pMem = pTree.getCumulativeRssmem();
long vMem = pTree.getCumulativeVmem();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java?rev=1428155&r1=1428154&r2=1428155&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java Thu Jan 3 03:42:15 2013
@@ -166,12 +166,10 @@ public class ProcfsBasedProcessTree exte
}
/**
- * Get the process-tree with latest state. If the root-process is not alive,
- * an empty tree will be returned.
- *
- * @return the process-tree with latest state.
+ * Update the process-tree with latest state. If the root-process is not alive,
+ * tree will become empty.
*/
- public ProcfsBasedProcessTree getProcessTree() {
+ public void updateProcessTree() {
if (!pid.equals(deadPid)) {
// Get the list of processes
List<String> processList = getProcessList();
@@ -197,7 +195,7 @@ public class ProcfsBasedProcessTree exte
}
if (me == null) {
- return this;
+ return;
}
// Add each process to its parent.
@@ -239,7 +237,6 @@ public class ProcfsBasedProcessTree exte
LOG.debug(this.toString());
}
}
- return this;
}
/**
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java?rev=1428155&r1=1428154&r2=1428155&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java Thu Jan 3 03:42:15 2013
@@ -151,7 +151,7 @@ public class TestProcfsBasedProcessTree
ProcfsBasedProcessTree p = new ProcfsBasedProcessTree(pid,
ProcessTree.isSetsidAvailable,
ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);
- p = p.getProcessTree(); // initialize
+ p.updateProcessTree(); // initialize
LOG.info("ProcessTree: " + p.toString());
File leaf = new File(lowestDescendant);
@@ -164,7 +164,7 @@ public class TestProcfsBasedProcessTree
}
}
- p = p.getProcessTree(); // reconstruct
+ p.updateProcessTree(); // reconstruct
LOG.info("ProcessTree: " + p.toString());
// Get the process-tree dump
@@ -203,7 +203,7 @@ public class TestProcfsBasedProcessTree
}
// ProcessTree is gone now. Any further calls should be sane.
- p = p.getProcessTree();
+ p.updateProcessTree();
assertFalse("ProcessTree must have been gone", p.isAlive());
assertTrue("Cumulative vmem for the gone-process is "
+ p.getCumulativeVmem() + " . It should be zero.", p
@@ -336,7 +336,7 @@ public class TestProcfsBasedProcessTree
new ProcfsBasedProcessTree("100", true, 100L,
procfsRootDir.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative memory
assertEquals("Cumulative virtual memory does not match", 600000L,
@@ -362,7 +362,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, pids, procInfos);
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative cpu time again
cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ?
@@ -409,7 +409,7 @@ public class TestProcfsBasedProcessTree
new ProcfsBasedProcessTree("100", true, 100L,
procfsRootDir.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// verify cumulative memory
assertEquals("Cumulative memory does not match",
@@ -425,7 +425,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, newPids, newProcInfos);
// check memory includes the new process.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
assertEquals("Cumulative vmem does not include new process",
1200000L, processTree.getCumulativeVmem());
long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
@@ -451,7 +451,7 @@ public class TestProcfsBasedProcessTree
writeStatFiles(procfsRootDir, newPids, newProcInfos);
// refresh process tree
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// processes older than 2 iterations should be same as before.
assertEquals("Cumulative vmem shouldn't have included new processes",
@@ -555,7 +555,7 @@ public class TestProcfsBasedProcessTree
new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir
.getAbsolutePath());
// build the process tree.
- processTree.getProcessTree();
+ processTree.updateProcessTree();
// Get the process-tree dump
String processTreeDump = processTree.getProcessTreeDump();