You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by yh...@apache.org on 2009/08/03 13:15:00 UTC
svn commit: r800306 - in /hadoop/mapreduce/trunk: ./
src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/
src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ src/java/
src/java/org/apache/hadoop/mapred/ src/test/mapred/org/ap...
Author: yhemanth
Date: Mon Aug 3 11:14:59 2009
New Revision: 800306
URL: http://svn.apache.org/viewvc?rev=800306&view=rev
Log:
MAPREDUCE-40. Keep memory management backwards compatible for job configuration parameters and limits. Contributed by Rahul Kumar Singh.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
hadoop/mapreduce/trunk/src/java/mapred-default.xml
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Aug 3 11:14:59 2009
@@ -305,3 +305,7 @@
MAPREDUCE-760. Fix a timing issue in TestNodeRefresh. (Amar Kamat via
sharad)
+ MAPREDUCE-40. Keep memory management backwards compatible for job
+ configuration parameters and limits. (Rahul Kumar Singh via yhemanth)
+
+
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java Mon Aug 3 11:14:59 2009
@@ -49,13 +49,16 @@
* {@link JobConf#setMaxVirtualMemoryForTask()}. This property thus provides
* default value of physical memory for job's that don't explicitly specify
* physical memory requirements.
- *
+ * <p/>
* It defaults to {@link JobConf#DISABLED_MEMORY_LIMIT} and if not explicitly
* set to a valid value, scheduler will not consider physical memory for
* scheduling even if virtual memory based scheduling is enabled.
+ *
+ * @deprecated
*/
+ @Deprecated
static String DEFAULT_PERCENTAGE_OF_PMEM_IN_VMEM_PROPERTY =
- "mapred.capacity-scheduler.task.default-pmem-percentage-in-vmem";
+ "mapred.capacity-scheduler.task.default-pmem-percentage-in-vmem";
/**
* Configuration that provides an upper limit on the maximum physical memory
@@ -65,9 +68,12 @@
* by the scheduler. If it is set to {@link JobConf#DISABLED_MEMORY_LIMIT},
* scheduler will not consider physical memory for scheduling even if virtual
* memory based scheduling is enabled.
+ *
+ * @deprecated
*/
+ @Deprecated
static final String UPPER_LIMIT_ON_TASK_PMEM_PROPERTY =
- "mapred.capacity-scheduler.task.limit.maxpmem";
+ "mapred.capacity-scheduler.task.limit.maxpmem";
/**
* Configuration that provides the maximum cap for the map task in a queue
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java Mon Aug 3 11:14:59 2009
@@ -931,6 +931,29 @@
}
private void initializeMemoryRelatedConf() {
+ //handling @deprecated
+ if (conf.get(
+ CapacitySchedulerConf.DEFAULT_PERCENTAGE_OF_PMEM_IN_VMEM_PROPERTY) !=
+ null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ CapacitySchedulerConf.DEFAULT_PERCENTAGE_OF_PMEM_IN_VMEM_PROPERTY));
+ }
+
+ //handling @deprecated
+ if (conf.get(CapacitySchedulerConf.UPPER_LIMIT_ON_TASK_PMEM_PROPERTY) !=
+ null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ CapacitySchedulerConf.UPPER_LIMIT_ON_TASK_PMEM_PROPERTY));
+ }
+
+ if (conf.get(JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY));
+ }
+
memSizeForMapSlotOnJT =
JobConf.normalizeMemoryConfigValue(conf.getLong(
JobTracker.MAPRED_CLUSTER_MAP_MEMORY_MB_PROPERTY,
@@ -939,14 +962,39 @@
JobConf.normalizeMemoryConfigValue(conf.getLong(
JobTracker.MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY,
JobConf.DISABLED_MEMORY_LIMIT));
- limitMaxMemForMapTasks =
- JobConf.normalizeMemoryConfigValue(conf.getLong(
+
+ //handling @deprecated values
+ if (conf.get(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY)+
+ " instead use " +JobTracker.MAPRED_CLUSTER_MAX_MAP_MEMORY_MB_PROPERTY+
+ " and " + JobTracker.MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB_PROPERTY
+ );
+
+ limitMaxMemForMapTasks = limitMaxMemForReduceTasks =
+ JobConf.normalizeMemoryConfigValue(
+ conf.getLong(
+ JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY,
+ JobConf.DISABLED_MEMORY_LIMIT));
+ if (limitMaxMemForMapTasks != JobConf.DISABLED_MEMORY_LIMIT &&
+ limitMaxMemForMapTasks >= 0) {
+ limitMaxMemForMapTasks = limitMaxMemForReduceTasks =
+ limitMaxMemForMapTasks /
+ (1024 * 1024); //Converting old values in bytes to MB
+ }
+ } else {
+ limitMaxMemForMapTasks =
+ JobConf.normalizeMemoryConfigValue(
+ conf.getLong(
JobTracker.MAPRED_CLUSTER_MAX_MAP_MEMORY_MB_PROPERTY,
JobConf.DISABLED_MEMORY_LIMIT));
- limitMaxMemForReduceTasks =
- JobConf.normalizeMemoryConfigValue(conf.getLong(
+ limitMaxMemForReduceTasks =
+ JobConf.normalizeMemoryConfigValue(
+ conf.getLong(
JobTracker.MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB_PROPERTY,
JobConf.DISABLED_MEMORY_LIMIT));
+ }
LOG.info(String.format("Scheduler configured with "
+ "(memSizeForMapSlotOnJT, memSizeForReduceSlotOnJT, "
+ "limitMaxMemForMapTasks, limitMaxMemForReduceTasks)"
@@ -1009,7 +1057,7 @@
}else {
totalCapacity += capacity;
}
- int ulMin = schedConf.getMinimumUserLimitPercent(queueName);
+ int ulMin = schedConf.getMinimumUserLimitPercent(queueName);
// create our QSI and add to our hashmap
QueueSchedulingInfo qsi = new QueueSchedulingInfo(
queueName, capacity, ulMin, jobQueuesManager, schedConf.getMaxMapCap(
Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacityScheduler.java Mon Aug 3 11:14:59 2009
@@ -47,7 +47,7 @@
LogFactory.getLog(org.apache.hadoop.mapred.TestCapacityScheduler.class);
private static int jobCounter;
-
+
/**
* Test class that removes the asynchronous nature of job initialization.
*
@@ -3022,4 +3022,22 @@
i++;
}
}
+
+ public void testDeprecatedMemoryValues() throws IOException {
+ // 2 map and 1 reduce slots
+ taskTrackerManager.addQueues(new String[] { "default" });
+ ArrayList<FakeQueueInfo> queues = new ArrayList<FakeQueueInfo>();
+ queues.add(new FakeQueueInfo("default", 100.0f, true, 25));
+ resConf.setFakeQueues(queues);
+ JobConf conf = (JobConf)(scheduler.getConf());
+ conf.set(
+ JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY, String.valueOf(
+ 1024 * 1024 * 3));
+ scheduler.setTaskTrackerManager(taskTrackerManager);
+ scheduler.setResourceManagerConf(resConf);
+ scheduler.start();
+
+ assertEquals(scheduler.getLimitMaxMemForMapSlot(),3);
+ assertEquals(scheduler.getLimitMaxMemForReduceSlot(),3);
+ }
}
Modified: hadoop/mapreduce/trunk/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/mapred-default.xml?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/mapred-default.xml (original)
+++ hadoop/mapreduce/trunk/src/java/mapred-default.xml Mon Aug 3 11:14:59 2009
@@ -173,76 +173,6 @@
</property>
<property>
- <name>mapred.tasktracker.vmem.reserved</name>
- <value>-1</value>
- <description>Configuration property to specify the amount of virtual memory
- that has to be reserved by the TaskTracker for system usage (OS, TT etc).
- The reserved virtual memory should be a part of the total virtual memory
- available on the TaskTracker.
-
- The reserved virtual memory and the total virtual memory values are
- reported by the TaskTracker as part of heart-beat so that they can
- considered by a scheduler. Please refer to the documentation of the
- configured scheduler to see how this property is used.
-
- These two values are also used by a TaskTracker for tracking tasks' memory
- usage. Memory management functionality on a TaskTracker is disabled if this
- property is set to -1, if it more than the total virtual memory on the
- tasktracker, or if either of the values is negative.
- </description>
-</property>
-
-<property>
- <name>mapred.task.limit.maxvmem</name>
- <value>-1</value>
- <description>
- Cluster-wide configuration in bytes to be set by the site administrators
- that provides an upper limit on the maximum virtual memory that can be
- specified by a job via mapred.task.maxvmem. This has to be set on both the
- JobTracker node for the sake of scheduling decisions and on the TaskTracker
- nodes for the sake of memory management.
-
- The job configuration mapred.task.maxvmem should not be more than this
- value, otherwise depending on the scheduler being configured, the job may
- be rejected or the job configuration may just be ignored. Please refer to
- the documentation of the configured scheduler to see how this property is
- used.
-
- If it is not set a TaskTracker, TaskTracker's memory management will be
- disabled.
- </description>
-</property>
-
-<property>
- <name>mapred.task.maxvmem</name>
- <value>-1</value>
- <description>
- The maximum amount of virtual memory any task of a job will use, in bytes.
-
- This value will be used by TaskTrackers for monitoring the memory usage of
- tasks of this jobs. If a TaskTracker's memory management functionality is
- enabled, each task of this job will be allowed to use a maximum virtual
- memory specified by this property. If the task's memory usage goes over
- this value, the task will be failed by the TT. If not set, the
- cluster-wide configuration mapred.task.default.maxvmem is used as the
- default value for memory requirements. If this property cascaded with
- mapred.task.default.maxvmem becomes equal to -1, the job's tasks will
- not be assured any particular amount of virtual memory and may be killed by
- a TT that intends to control the total memory usage of the tasks via memory
- management functionality. If the memory management functionality is
- disabled on a TT, this value is ignored.
-
- This value should not be more than the cluster-wide configuration
- mapred.task.limit.maxvmem.
-
- This value may be used by schedulers that support scheduling based on job's
- memory requirements. Please refer to the documentation of the scheduler
- being configured to see if it does memory based scheduling and if it does,
- how this property is used by that scheduler.
- </description>
-</property>
-
-<property>
<name>mapred.tasktracker.memory_calculator_plugin</name>
<value></value>
<description>
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Mon Aug 3 11:14:59 2009
@@ -112,6 +112,34 @@
}
/**
+ * @deprecated
+ */
+ @Deprecated
+ public static final String MAPRED_TASK_MAXVMEM_PROPERTY =
+ "mapred.task.maxvmem";
+
+ /**
+ * @deprecated
+ */
+ @Deprecated
+ public static final String UPPER_LIMIT_ON_TASK_VMEM_PROPERTY =
+ "mapred.task.limit.maxvmem";
+
+ /**
+ * @deprecated
+ */
+ @Deprecated
+ public static final String MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY =
+ "mapred.task.default.maxvmem";
+
+ /**
+ * @deprecated
+ */
+ @Deprecated
+ public static final String MAPRED_TASK_MAXPMEM_PROPERTY =
+ "mapred.task.maxpmem";
+
+ /**
* A value which if set for memory related configuration options,
* indicates that the options are turned off.
*/
@@ -1367,7 +1395,8 @@
*
* @param uri the job end notification uri
* @see JobStatus
- * @see <a href="{@docRoot}/org/apache/hadoop/mapred/JobClient.html#JobCompletionAndChaining">Job Completion and Chaining</a>
+ * @see <a href="{@docRoot}/org/apache/hadoop/mapred/JobClient.html#
+ * JobCompletionAndChaining">Job Completion and Chaining</a>
*/
public void setJobEndNotificationURI(String uri) {
set("job.end.notification.url", uri);
@@ -1392,21 +1421,46 @@
return get("job.local.dir");
}
- long getMemoryForMapTask() {
- return getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
- DISABLED_MEMORY_LIMIT);
+ public long getMemoryForMapTask() {
+ if (get(MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ JobConf.MAPRED_TASK_MAXVMEM_PROPERTY)+
+ " instead use "+JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY + " and "
+ + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
+
+ long val = getLong(
+ MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
+ return (val == DISABLED_MEMORY_LIMIT) ? val :
+ ((val < 0) ? DISABLED_MEMORY_LIMIT : val / (1024 * 1024));
+ }
+ return getLong(
+ JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
+ DISABLED_MEMORY_LIMIT);
}
- void setMemoryForMapTask(long mem) {
+ public void setMemoryForMapTask(long mem) {
setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, mem);
}
- long getMemoryForReduceTask() {
- return getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
- DISABLED_MEMORY_LIMIT);
+ public long getMemoryForReduceTask() {
+ if (get(MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ JobConf.MAPRED_TASK_MAXVMEM_PROPERTY)+
+ " instead use "+JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY + " and "
+ + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
+ long val = getLong(
+ MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
+ return (val == DISABLED_MEMORY_LIMIT) ? val :
+ ((val < 0) ? DISABLED_MEMORY_LIMIT : val / (1024 * 1024));
+ }
+ return getLong(
+ JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
+ DISABLED_MEMORY_LIMIT);
}
- void setMemoryForReduceTask(long mem) {
+ public void setMemoryForReduceTask(long mem) {
setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, mem);
}
@@ -1505,5 +1559,110 @@
}
return null;
}
+
+
+ /**
+ * The maximum amount of memory any task of this job will use. See
+ * {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
+ * <p/>
+ * mapred.task.maxvmem is split into
+ * mapred.job.map.memory.mb
+ * and mapred.job.map.memory.mb,mapred
+ * each of the new key are set
+ * as mapred.task.maxvmem / 1024
+ * as new values are in MB
+ *
+ * @return The maximum amount of memory any task of this job will use, in
+ * bytes.
+ * @see #setMaxVirtualMemoryForTask(long)
+ * @deprecated Use {@link #getMemoryForMapTask()} and
+ * {@link #getMemoryForReduceTask()}
+ */
+ @Deprecated
+ public long getMaxVirtualMemoryForTask() {
+ LOG.warn(
+ "getMaxVirtualMemoryForTask() is deprecated. " +
+ "Instead use getMemoryForMapTask() and getMemoryForReduceTask()");
+
+ if (get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) == null) {
+ if (get(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY) != null || get(
+ JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY) != null) {
+ long val = Math.max(getMemoryForMapTask(), getMemoryForReduceTask());
+ if (val == JobConf.DISABLED_MEMORY_LIMIT) {
+ return val;
+ } else {
+ if (val < 0) {
+ return JobConf.DISABLED_MEMORY_LIMIT;
+ }
+ return val * 1024 * 1024;
+ //Convert MB to byte as new value is in
+ // MB and old deprecated method returns bytes
+ }
+ }
+ }
+
+ return getLong(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
+ }
+
+ /**
+ * Set the maximum amount of memory any task of this job can use. See
+ * {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
+ * <p/>
+ * mapred.task.maxvmem is split into
+ * mapred.job.map.memory.mb
+ * and mapred.job.map.memory.mb,mapred
+ * each of the new key are set
+ * as mapred.task.maxvmem / 1024
+ * as new values are in MB
+ *
+ * @param vmem Maximum amount of virtual memory in bytes any task of this job
+ * can use.
+ * @see #getMaxVirtualMemoryForTask()
+ * @deprecated
+ * Use {@link #setMemoryForMapTask(long mem)} and
+ * Use {@link #setMemoryForReduceTask(long mem)}
+ */
+ @Deprecated
+ public void setMaxVirtualMemoryForTask(long vmem) {
+ LOG.warn("setMaxVirtualMemoryForTask() is deprecated."+
+ "Instead use setMemoryForMapTask() and setMemoryForReduceTask()");
+ if(vmem != DISABLED_MEMORY_LIMIT && vmem < 0) {
+ setMemoryForMapTask(DISABLED_MEMORY_LIMIT);
+ setMemoryForReduceTask(DISABLED_MEMORY_LIMIT);
+ }
+
+ if(get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) == null) {
+ setMemoryForMapTask(vmem / (1024 * 1024)); //Changing bytes to mb
+ setMemoryForReduceTask(vmem / (1024 * 1024));//Changing bytes to mb
+ }else{
+ this.setLong(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY,vmem);
+ }
+ }
+
+ /**
+ * @deprecated this variable is deprecated and nolonger in use.
+ */
+ @Deprecated
+ public long getMaxPhysicalMemoryForTask() {
+ LOG.warn("The API getMaxPhysicalMemoryForTask() is deprecated."
+ + " Refer to the APIs getMemoryForMapTask() and"
+ + " getMemoryForReduceTask() for details.");
+ return -1;
+ }
+
+ /*
+ * @deprecated this
+ */
+ @Deprecated
+ public void setMaxPhysicalMemoryForTask(long mem) {
+ LOG.warn("The API setMaxPhysicalMemoryForTask() is deprecated."
+ + " The value set is ignored. Refer to "
+ + " setMemoryForMapTask() and setMemoryForReduceTask() for details.");
+ }
+
+ static String deprecatedString(String key) {
+ return "The variable " + key + " is no longer used";
+ }
+
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Aug 3 11:14:59 2009
@@ -4194,14 +4194,39 @@
JobConf.normalizeMemoryConfigValue(conf.getLong(
JobTracker.MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY,
JobConf.DISABLED_MEMORY_LIMIT));
- limitMaxMemForMapTasks =
- JobConf.normalizeMemoryConfigValue(conf.getLong(
+
+ if (conf.get(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY)+
+ " instead use "+JobTracker.MAPRED_CLUSTER_MAX_MAP_MEMORY_MB_PROPERTY+
+ " and " + JobTracker.MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB_PROPERTY
+ );
+
+ limitMaxMemForMapTasks = limitMaxMemForReduceTasks =
+ JobConf.normalizeMemoryConfigValue(
+ conf.getLong(
+ JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY,
+ JobConf.DISABLED_MEMORY_LIMIT));
+ if (limitMaxMemForMapTasks != JobConf.DISABLED_MEMORY_LIMIT &&
+ limitMaxMemForMapTasks >= 0) {
+ limitMaxMemForMapTasks = limitMaxMemForReduceTasks =
+ limitMaxMemForMapTasks /
+ (1024 * 1024); //Converting old values in bytes to MB
+ }
+ } else {
+ limitMaxMemForMapTasks =
+ JobConf.normalizeMemoryConfigValue(
+ conf.getLong(
JobTracker.MAPRED_CLUSTER_MAX_MAP_MEMORY_MB_PROPERTY,
JobConf.DISABLED_MEMORY_LIMIT));
- limitMaxMemForReduceTasks =
- JobConf.normalizeMemoryConfigValue(conf.getLong(
+ limitMaxMemForReduceTasks =
+ JobConf.normalizeMemoryConfigValue(
+ conf.getLong(
JobTracker.MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB_PROPERTY,
JobConf.DISABLED_MEMORY_LIMIT));
+ }
+
LOG.info(new StringBuilder().append("Scheduler configured with ").append(
"(memSizeForMapSlotOnJT, memSizeForReduceSlotOnJT,").append(
" limitMaxMemForMapTasks, limitMaxMemForReduceTasks) (").append(
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Mon Aug 3 11:14:59 2009
@@ -98,7 +98,19 @@
*******************************************************/
public class TaskTracker
implements MRConstants, TaskUmbilicalProtocol, Runnable {
-
+ /**
+ * @deprecated
+ */
+ @Deprecated
+ static final String MAPRED_TASKTRACKER_VMEM_RESERVED_PROPERTY =
+ "mapred.tasktracker.vmem.reserved";
+ /**
+ * @deprecated
+ */
+ @Deprecated
+ static final String MAPRED_TASKTRACKER_PMEM_RESERVED_PROPERTY =
+ "mapred.tasktracker.pmem.reserved";
+
static final long WAIT_FOR_DONE = 3 * 1000;
private int httpPort;
@@ -551,13 +563,15 @@
startHealthMonitor(this.fConf);
}
}
-
- public static Class<? extends TaskTrackerInstrumentation> getInstrumentationClass(Configuration conf) {
+
+ public static Class<? extends TaskTrackerInstrumentation> getInstrumentationClass(
+ Configuration conf) {
return conf.getClass("mapred.tasktracker.instrumentation",
TaskTrackerMetricsInst.class, TaskTrackerInstrumentation.class);
}
-
- public static void setInstrumentationClass(Configuration conf, Class<? extends TaskTrackerInstrumentation> t) {
+
+ public static void setInstrumentationClass(
+ Configuration conf, Class<? extends TaskTrackerInstrumentation> t) {
conf.setClass("mapred.tasktracker.instrumentation",
t, TaskTrackerInstrumentation.class);
}
@@ -3059,6 +3073,35 @@
* Memory-related setup
*/
private void initializeMemoryManagement() {
+
+ //handling @deprecated
+ if (fConf.get(MAPRED_TASKTRACKER_VMEM_RESERVED_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ MAPRED_TASKTRACKER_VMEM_RESERVED_PROPERTY));
+ }
+
+ //handling @deprecated
+ if (fConf.get(MAPRED_TASKTRACKER_PMEM_RESERVED_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ MAPRED_TASKTRACKER_PMEM_RESERVED_PROPERTY));
+ }
+
+ //handling @deprecated
+ if (fConf.get(JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY));
+ }
+
+ //handling @deprecated
+ if (fConf.get(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY) != null) {
+ LOG.warn(
+ JobConf.deprecatedString(
+ JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY));
+ }
+
Class<? extends MemoryCalculatorPlugin> clazz =
fConf.getClass(MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
null, MemoryCalculatorPlugin.class);
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java?rev=800306&r1=800305&r2=800306&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java Mon Aug 3 11:14:59 2009
@@ -49,4 +49,93 @@
configuration.set("mapred.task.profile.params", "test");
Assert.assertEquals("test", configuration.getProfileParams());
}
+
+ /**
+ * Testing mapred.task.maxvmem replacement with new values
+ *
+ */
+ public void testMemoryConfigForMapOrReduceTask(){
+ JobConf configuration = new JobConf();
+ configuration.set("mapred.job.map.memory.mb",String.valueOf(300));
+ configuration.set("mapred.job.reduce.memory.mb",String.valueOf(300));
+ Assert.assertEquals(configuration.getMemoryForMapTask(),300);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),300);
+
+ configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+ configuration.set("mapred.job.map.memory.mb",String.valueOf(300));
+ configuration.set("mapred.job.reduce.memory.mb",String.valueOf(300));
+ Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem" , "-1");
+ configuration.set("mapred.job.map.memory.mb",String.valueOf(300));
+ configuration.set("mapred.job.reduce.memory.mb",String.valueOf(300));
+ Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+ configuration.set("mapred.job.map.memory.mb","-1");
+ configuration.set("mapred.job.reduce.memory.mb","-1");
+ Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
+ configuration.set("mapred.job.map.memory.mb","-1");
+ configuration.set("mapred.job.reduce.memory.mb","-1");
+ Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+ Assert.assertEquals(configuration.getMemoryForMapTask(),2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+ }
+
+ /**
+ * Test deprecated accessor and mutator method for mapred.task.maxvmem
+ */
+ public void testMaxVirtualMemoryForTask() {
+ JobConf configuration = new JobConf();
+
+ //get test case
+ configuration.set("mapred.job.map.memory.mb", String.valueOf(300));
+ configuration.set("mapred.job.reduce.memory.mb", String.valueOf(-1));
+ Assert.assertEquals(
+ configuration.getMaxVirtualMemoryForTask(), 300 * 1024 * 1024);
+
+ configuration = new JobConf();
+ configuration.set("mapred.job.map.memory.mb", String.valueOf(-1));
+ configuration.set("mapred.job.reduce.memory.mb", String.valueOf(200));
+ Assert.assertEquals(
+ configuration.getMaxVirtualMemoryForTask(), 200 * 1024 * 1024);
+
+ configuration = new JobConf();
+ configuration.set("mapred.job.map.memory.mb", String.valueOf(-1));
+ configuration.set("mapred.job.reduce.memory.mb", String.valueOf(-1));
+ configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
+ Assert.assertEquals(
+ configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
+
+ configuration = new JobConf();
+ configuration.set("mapred.task.maxvmem", String.valueOf(1 * 1024 * 1024));
+ Assert.assertEquals(
+ configuration.getMaxVirtualMemoryForTask(), 1 * 1024 * 1024);
+
+ //set test case
+
+ configuration = new JobConf();
+ configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
+ Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
+
+ configuration = new JobConf();
+ configuration.set("mapred.job.map.memory.mb", String.valueOf(300));
+ configuration.set("mapred.job.reduce.memory.mb", String.valueOf(400));
+ configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
+ Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
+ }
}