You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2008/09/16 20:43:52 UTC
svn commit: r696002 - in /hadoop/core/trunk: ./ conf/
src/core/org/apache/hadoop/util/ src/mapred/org/apache/hadoop/mapred/
src/test/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/util/
Author: acmurthy
Date: Tue Sep 16 11:43:51 2008
New Revision: 696002
URL: http://svn.apache.org/viewvc?rev=696002&view=rev
Log:
HADOOP-4129. Changed memory limits of TaskTracker and Tasks to be in KiloBytes rather than bytes. Contributed by Vinod Kumar Vavilapalli.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/conf/hadoop-default.xml
hadoop/core/trunk/src/core/org/apache/hadoop/util/ProcfsBasedProcessTree.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTrackerStatus.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
hadoop/core/trunk/src/test/org/apache/hadoop/util/TestProcfsBasedProcessTree.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Sep 16 11:43:51 2008
@@ -562,6 +562,9 @@
HADOOP-3570. Includes user specified libjar files in the client side
classpath path. (Sharad Agarwal via ddas)
+ HADOOP-4129. Changed memory limits of TaskTracker and Tasks to be in
+ KiloBytes rather than bytes. (Vinod Kumar Vavilapalli via acmurthy)
+
Release 0.18.1 - 2008-09-17
IMPROVEMENTS
Modified: hadoop/core/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/hadoop-default.xml?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/conf/hadoop-default.xml (original)
+++ hadoop/core/trunk/conf/hadoop-default.xml Tue Sep 16 11:43:51 2008
@@ -1424,22 +1424,22 @@
<property>
<name>mapred.tasktracker.tasks.maxmemory</name>
<value>-1</value>
- <description> The maximum amount of virtual memory all tasks running on a
- tasktracker, including sub-processes they launch, can use. This value is
- used to compute the amount of free memory available for tasks. Any task
- scheduled on this tasktracker is guaranteed and constrained to use a
- share of this amount. Any task exceeding its share will be killed.
- If set to -1, this functionality is disabled, and mapred.task.maxmemory
- is ignored.
+ <description> The maximum amount of virtual memory in kilobytes all tasks
+ running on a tasktracker, including sub-processes they launch, can use.
+ This value is used to compute the amount of free memory available for
+ tasks. Any task scheduled on this tasktracker is guaranteed and constrained
+ to use a share of this amount. Any task exceeding its share will be
+ killed. If set to -1, this functionality is disabled, and
+ mapred.task.maxmemory is ignored.
</description>
</property>
<property>
<name>mapred.task.maxmemory</name>
<value>-1</value>
- <description> The maximum amount of memory any task of a job will use.
- A task of this job will be scheduled on a tasktracker, only if the
- amount of free memory on the tasktracker is greater than or
+ <description> The maximum amount of memory in kilobytes any task of a job
+ will use. A task of this job will be scheduled on a tasktracker, only if
+ the amount of free memory on the tasktracker is greater than or
equal to this value. If set to -1, tasks are assured a memory limit on
the tasktracker equal to
mapred.tasktracker.tasks.maxmemory/number of slots. If the value of
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/util/ProcfsBasedProcessTree.java?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/util/ProcfsBasedProcessTree.java Tue Sep 16 11:43:51 2008
@@ -189,7 +189,7 @@
* Get the cumulative virtual memory used by all the processes in the
* process-tree.
*
- * @return cumulative virtual memory used by the process-tree in bytes.
+ * @return cumulative virtual memory used by the process-tree in kilobytes.
*/
public long getCumulativeVmem() {
long total = 0;
@@ -198,7 +198,7 @@
total += p.getVmem();
}
}
- return total;
+ return total/1024;
}
/**
@@ -466,4 +466,4 @@
return children;
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobConf.java Tue Sep 16 11:43:51 2008
@@ -1331,7 +1331,7 @@
* If set to {@link #DISABLED_VIRTUAL_MEMORY_LIMIT}, this functionality
* is disabled.
*
- * @return maximum amount of virtual memory to divide among
+ * @return maximum amount of virtual memory in kilobytes to divide among
* @see #getMaxVirtualMemoryForTask()
*/
public long getMaxVirtualMemoryForTasks() {
@@ -1343,7 +1343,7 @@
* Set the maximum amount of virtual memory all tasks running on a
* tasktracker, including sub-processes they launch, can use.
*
- * @param vmem maximum amount of virtual memory that can be used.
+ * @param vmem maximum amount of virtual memory in kilobytes that can be used.
* @see #getMaxVirtualMemoryForTasks()
*/
public void setMaxVirtualMemoryForTasks(long vmem) {
@@ -1363,7 +1363,7 @@
* mapred.tasktracker.tasks.maxmemory is set to -1, this value is
* ignored.
*
- * @return The maximum amount of memory any task of this job will use.
+ * @return The maximum amount of memory any task of this job will use, in kilobytes.
* @see #getMaxVirtualMemoryForTasks()
*/
public long getMaxVirtualMemoryForTask() {
@@ -1373,7 +1373,8 @@
/**
* Set the maximum amount of memory any task of this job can use.
*
- * @param vmem Maximum amount of memory any task of this job can use.
+ * @param vmem Maximum amount of memory in kilobytes any task of this job
+ * can use.
* @see #getMaxVirtualMemoryForTask()
*/
public void setMaxVirtualMemoryForTask(long vmem) {
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskMemoryManagerThread.java?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskMemoryManagerThread.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskMemoryManagerThread.java Tue Sep 16 11:43:51 2008
@@ -196,7 +196,7 @@
long currentMemUsage = pTree.getCumulativeVmem();
long limit = ptInfo.getMemLimit();
LOG.info("Memory usage of ProcessTree " + pId + " :" + currentMemUsage
- + ". Limit : " + limit);
+ + "kB. Limit : " + limit + "kB");
if (limit != JobConf.DISABLED_VIRTUAL_MEMORY_LIMIT
&& currentMemUsage > limit) {
@@ -204,7 +204,7 @@
// Clean up.
String msg = "TaskTree [pid=" + pId + ",tipID=" + tid
+ "] is running beyond memory-limits. Current usage : "
- + currentMemUsage + ". Limit : " + limit + ". Killing task.";
+ + currentMemUsage + "kB. Limit : " + limit + "kB. Killing task.";
LOG.warn(msg);
taskTracker.cleanUpOverMemoryTask(tid, msg);
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Tue Sep 16 11:43:51 2008
@@ -1192,7 +1192,7 @@
/**
* Return the maximum amount of memory available for all tasks on
* this tracker
- * @return maximum amount of virtual memory
+ * @return maximum amount of virtual memory in kilobytes
*/
long getMaxVirtualMemoryForTasks() {
return maxVirtualMemoryForTasks;
@@ -1208,7 +1208,7 @@
* and the total amount of maximum virtual memory that can be
* used by all currently running tasks.
*
- * @return amount of free virtual memory that can be assured for
+ * @return amount of free virtual memory in kilobytes that can be assured for
* new tasks
*/
private synchronized long findFreeVirtualMemory() {
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTrackerStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTrackerStatus.java?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTrackerStatus.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTrackerStatus.java Tue Sep 16 11:43:51 2008
@@ -67,7 +67,7 @@
/**
* Set the amount of free virtual memory that is available for running
* a new task
- * @param freeVMem amount of free virtual memory
+ * @param freeVMem amount of free virtual memory in kilobytes
*/
void setFreeVirtualMemory(long freeVmem) {
freeVirtualMemory = freeVmem;
@@ -80,7 +80,7 @@
* If this is {@link JobConf.DISABLED_VIRTUAL_MEMORY_LIMIT}, it should
* be ignored and not used in computation.
*
- *@return amount of free virtual memory.
+ *@return amount of free virtual memory in kilobytes.
*/
long getFreeVirtualMemory() {
return freeVirtualMemory;
@@ -88,7 +88,7 @@
/**
* Set the default amount of virtual memory per task.
- * @param vmem amount of free virtual memory.
+ * @param vmem amount of free virtual memory in kilobytes.
*/
void setDefaultVirtualMemoryPerTask(long defaultVmem) {
defaultVirtualMemoryPerTask = defaultVmem;
@@ -102,7 +102,7 @@
* {@link JobConf.DISABLED_VIRTUAL_MEMORY_LIMIT}, it should be ignored
* and not used in any computation.
*
- * @return default amount of virtual memory per task.
+ * @return default amount of virtual memory per task in kilobytes.
*/
long getDefaultVirtualMemoryPerTask() {
return defaultVirtualMemoryPerTask;
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java Tue Sep 16 11:43:51 2008
@@ -98,7 +98,7 @@
Pattern diagMsgPattern = Pattern
.compile("TaskTree \\[pid=[0-9]*,tipID=.*\\] is running beyond "
- + "memory-limits. Current usage : [0-9]*. Limit : [0-9]*. Killing task.");
+ + "memory-limits. Current usage : [0-9]*kB. Limit : [0-9]*kB. Killing task.");
Matcher mat = null;
// Start cluster with proper configuration.
@@ -172,8 +172,8 @@
long PER_TASK_LIMIT = 444; // Enough to kill off WordCount.
Pattern diagMsgPattern = Pattern
.compile("TaskTree \\[pid=[0-9]*,tipID=.*\\] is running beyond "
- + "memory-limits. Current usage : [0-9]*. Limit : "
- + PER_TASK_LIMIT + ". Killing task.");
+ + "memory-limits. Current usage : [0-9]*kB. Limit : "
+ + PER_TASK_LIMIT + "kB. Killing task.");
Matcher mat = null;
// Start cluster with proper configuration.
@@ -235,4 +235,4 @@
}
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/util/TestProcfsBasedProcessTree.java?rev=696002&r1=696001&r2=696002&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/util/TestProcfsBasedProcessTree.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/util/TestProcfsBasedProcessTree.java Tue Sep 16 11:43:51 2008
@@ -39,7 +39,7 @@
private String shellScript;
private static final int N = 10; // Controls the RogueTask
- private static final int memoryLimit = 15000000; // bytes
+ private static final int memoryLimit = 15000; // kilobytes
private static final long PROCESSTREE_RECONSTRUCTION_INTERVAL =
ProcfsBasedProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL; // msec
@@ -125,7 +125,7 @@
while (true) {
LOG.info("ProcessTree: " + p.toString());
long mem = p.getCumulativeVmem();
- LOG.info("Memory usage: " + mem + "bytes.");
+ LOG.info("Memory usage: " + mem + "kB.");
if (mem > memoryLimit) {
p.destroy();
break;
@@ -146,4 +146,4 @@
LOG.info("Interrupted while joining RogueTaskThread.");
}
}
-}
\ No newline at end of file
+}