You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yh...@apache.org on 2009/08/21 07:34:27 UTC
svn commit: r806422 - in /hadoop/common/branches/branch-0.20: CHANGES.txt
src/mapred/org/apache/hadoop/mapred/TaskTracker.java
src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
Author: yhemanth
Date: Fri Aug 21 05:34:26 2009
New Revision: 806422
URL: http://svn.apache.org/viewvc?rev=806422&view=rev
Log:
MAPREDUCE-834. Enables memory management on tasktrackers when old memory management parameters are used in configuration. Contributed by Sreekanth Ramakrishnan.
Modified:
hadoop/common/branches/branch-0.20/CHANGES.txt
hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
Modified: hadoop/common/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/CHANGES.txt?rev=806422&r1=806421&r2=806422&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20/CHANGES.txt Fri Aug 21 05:34:26 2009
@@ -221,6 +221,10 @@
MAPREDUCE-745. Fixes a testcase problem to do with generation of JobTracker
IDs. (Amar Kamat via ddas)
+ MAPREDUCE-834. Enables memory management on tasktrackers when old
+ memory management parameters are used in configuration.
+ (Sreekanth Ramakrishnan via yhemanth)
+
Release 0.20.0 - 2009-04-15
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=806422&r1=806421&r2=806422&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/common/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Fri Aug 21 05:34:26 2009
@@ -3081,7 +3081,26 @@
maxCurrentMapTasks * mapSlotMemorySizeOnTT + maxCurrentReduceTasks
* reduceSlotSizeMemoryOnTT;
if (totalMemoryAllottedForTasks < 0) {
- totalMemoryAllottedForTasks = JobConf.DISABLED_MEMORY_LIMIT;
+ //adding check for the old keys which might be used by the administrator
+ //while configuration of the memory monitoring on TT
+ long memoryAllotedForSlot = fConf.normalizeMemoryConfigValue(
+ fConf.getLong(JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY,
+ JobConf.DISABLED_MEMORY_LIMIT));
+ long limitVmPerTask = fConf.normalizeMemoryConfigValue(
+ fConf.getLong(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY,
+ JobConf.DISABLED_MEMORY_LIMIT));
+ if(memoryAllotedForSlot == JobConf.DISABLED_MEMORY_LIMIT) {
+ totalMemoryAllottedForTasks = JobConf.DISABLED_MEMORY_LIMIT;
+ } else {
+ if(memoryAllotedForSlot > limitVmPerTask) {
+ LOG.info("DefaultMaxVmPerTask is mis-configured. " +
+ "It shouldn't be greater than task limits");
+ totalMemoryAllottedForTasks = JobConf.DISABLED_MEMORY_LIMIT;
+ } else {
+ totalMemoryAllottedForTasks = (maxCurrentMapTasks +
+ maxCurrentReduceTasks) * (memoryAllotedForSlot/(1024 * 1024));
+ }
+ }
}
if (totalMemoryAllottedForTasks > totalPhysicalMemoryOnTT) {
LOG.info("totalMemoryAllottedForTasks > totalPhysicalMemoryOnTT."
Modified: hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java?rev=806422&r1=806421&r2=806422&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java (original)
+++ hadoop/common/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java Fri Aug 21 05:34:26 2009
@@ -200,16 +200,8 @@
return;
}
- long PER_TASK_LIMIT = 1L; // Low enough to kill off sleepJob tasks.
-
- Pattern taskOverLimitPattern =
- Pattern.compile(String.format(taskOverLimitPatternString, String
- .valueOf(PER_TASK_LIMIT*1024*1024L)));
- Matcher mat = null;
-
// Start cluster with proper configuration.
JobConf fConf = new JobConf();
-
// very small value, so that no task escapes to successful completion.
fConf.set("mapred.tasktracker.taskmemorymanager.monitoring-interval",
String.valueOf(300));
@@ -219,6 +211,51 @@
JobTracker.MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY,
2 * 1024);
startCluster(fConf);
+ runJobExceedingMemoryLimit();
+ }
+
+ /**
+ * Runs tests with tasks beyond limit and using old configuration values for
+ * the TaskTracker.
+ *
+ * @throws Exception
+ */
+
+ public void testTaskMemoryMonitoringWithDeprecatedConfiguration ()
+ throws Exception {
+
+ // Run the test only if memory management is enabled
+ if (!isProcfsBasedTreeAvailable()) {
+ return;
+ }
+ // Start cluster with proper configuration.
+ JobConf fConf = new JobConf();
+ // very small value, so that no task escapes to successful completion.
+ fConf.set("mapred.tasktracker.taskmemorymanager.monitoring-interval",
+ String.valueOf(300));
+ //set old values, max vm property per task and upper limit on the tasks
+ //vm
+ //setting the default maximum vmem property to 2 GB
+ fConf.setLong(JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY,
+ (2L * 1024L * 1024L * 1024L));
+ fConf.setLong(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY,
+ (3L * 1024L * 1024L * 1024L));
+ startCluster(fConf);
+ runJobExceedingMemoryLimit();
+ }
+
+ /**
+ * Runs a job which should fail the when run by the memory monitor.
+ *
+ * @throws IOException
+ */
+ private void runJobExceedingMemoryLimit() throws IOException {
+ long PER_TASK_LIMIT = 1L; // Low enough to kill off sleepJob tasks.
+
+ Pattern taskOverLimitPattern =
+ Pattern.compile(String.format(taskOverLimitPatternString, String
+ .valueOf(PER_TASK_LIMIT*1024*1024L)));
+ Matcher mat = null;
// Set up job.
JobConf conf = new JobConf(miniMRCluster.createJobConf());