You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by yh...@apache.org on 2009/10/12 11:30:49 UTC
svn commit: r824273 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/JobConf.java
src/test/mapred/org/apache/hadoop/conf/TestJobConf.java
Author: yhemanth
Date: Mon Oct 12 09:30:49 2009
New Revision: 824273
URL: http://svn.apache.org/viewvc?rev=824273&view=rev
Log:
MAPREDUCE-979. Fixed JobConf APIs related to memory parameters to return values of new configuration variables when deprecated variables are disabled. Contributed by Sreekanth Ramakrishnan.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=824273&r1=824272&r2=824273&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Oct 12 09:30:49 2009
@@ -745,3 +745,7 @@
MAPREDUCE-1076. Deprecate ClusterStatus and add javadoc in ClusterMetrics.
(Amareshwari Sriramadasu via sharad)
+ MAPREDUCE-979. Fixed JobConf APIs related to memory parameters to return
+ values of new configuration variables when deprecated variables are
+ disabled. (Sreekanth Ramakrishnan via yhemanth)
+
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=824273&r1=824272&r2=824273&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Mon Oct 12 09:30:49 2009
@@ -42,8 +42,6 @@
import org.apache.hadoop.mapred.lib.KeyFieldBasedComparator;
import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner;
import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
import org.apache.hadoop.mapreduce.util.ConfigUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
@@ -116,7 +114,8 @@
}
/**
- * @deprecated
+ * @deprecated Use {@link #MAPRED_JOB_MAP_MEMORY_MB_PROPERTY} and
+ * {@link #MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY}
*/
@Deprecated
public static final String MAPRED_TASK_MAXVMEM_PROPERTY =
@@ -1608,32 +1607,68 @@
return get(JobContext.JOB_LOCAL_DIR);
}
+ /**
+ * Get memory required to run a map task of the job, in MB.
+ *
+ * If a value is specified in the configuration, it is returned.
+ * Else, it returns {@link #DISABLED_MEMORY_LIMIT}.
+ * <p/>
+ * For backward compatibility, if the job configuration sets the
+ * key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ * from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ * after converting it from bytes to MB.
+ * @return memory required to run a map task of the job, in MB,
+ * or {@link #DISABLED_MEMORY_LIMIT} if unset.
+ */
public long getMemoryForMapTask() {
- if (get(MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
- long val = getLong(
- MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
- return (val == DISABLED_MEMORY_LIMIT) ? val :
- ((val < 0) ? DISABLED_MEMORY_LIMIT : val / (1024 * 1024));
+ long value = getDeprecatedMemoryValue();
+ if (value == DISABLED_MEMORY_LIMIT) {
+ value = normalizeMemoryConfigValue(
+ getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
+ DISABLED_MEMORY_LIMIT));
}
- return getLong(
- JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
- DISABLED_MEMORY_LIMIT);
+ return value;
}
public void setMemoryForMapTask(long mem) {
setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, mem);
}
+ /**
+ * Get memory required to run a reduce task of the job, in MB.
+ *
+ * If a value is specified in the configuration, it is returned.
+ * Else, it returns {@link #DISABLED_MEMORY_LIMIT}.
+ * <p/>
+ * For backward compatibility, if the job configuration sets the
+ * key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ * from {@link #DISABLED_MEMORY_LIMIT}, that value will be used
+ * after converting it from bytes to MB.
+ * @return memory required to run a reduce task of the job, in MB,
+ * or {@link #DISABLED_MEMORY_LIMIT} if unset.
+ */
public long getMemoryForReduceTask() {
- if (get(MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
- long val = getLong(
- MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
- return (val == DISABLED_MEMORY_LIMIT) ? val :
- ((val < 0) ? DISABLED_MEMORY_LIMIT : val / (1024 * 1024));
+ long value = getDeprecatedMemoryValue();
+ if (value == DISABLED_MEMORY_LIMIT) {
+ value = normalizeMemoryConfigValue(
+ getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
+ DISABLED_MEMORY_LIMIT));
+ }
+ return value;
+ }
+
+ // Return the value set to the key MAPRED_TASK_MAXVMEM_PROPERTY,
+ // converted into MBs.
+ // Returns DISABLED_MEMORY_LIMIT if unset, or set to a negative
+ // value.
+ private long getDeprecatedMemoryValue() {
+ long oldValue = getLong(MAPRED_TASK_MAXVMEM_PROPERTY,
+ DISABLED_MEMORY_LIMIT);
+ oldValue = normalizeMemoryConfigValue(oldValue);
+ if (oldValue != DISABLED_MEMORY_LIMIT) {
+ oldValue /= (1024*1024);
}
- return getLong(
- JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
- DISABLED_MEMORY_LIMIT);
+ return oldValue;
}
public void setMemoryForReduceTask(long mem) {
@@ -1738,18 +1773,21 @@
/**
- * The maximum amount of memory any task of this job will use. See
+ * Get the memory required to run a task of this job, in bytes. See
* {@link #MAPRED_TASK_MAXVMEM_PROPERTY}
* <p/>
- * mapred.task.maxvmem is split into
- * mapreduce.map.memory.mb
- * and mapreduce.map.memory.mb,mapred
- * each of the new key are set
- * as mapred.task.maxvmem / 1024
- * as new values are in MB
+ * This method is deprecated. Now, different memory limits can be
+ * set for map and reduce tasks of a job, in MB.
+ * <p/>
+ * For backward compatibility, if the job configuration sets the
+ * key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different
+ * from {@link #DISABLED_MEMORY_LIMIT}, that value is returned.
+ * Otherwise, this method will return the larger of the values returned by
+ * {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()}
+ * after converting them into bytes.
*
- * @return The maximum amount of memory any task of this job will use, in
- * bytes.
+ * @return Memory required to run a task of this job, in bytes,
+ * or {@link #DISABLED_MEMORY_LIMIT}, if unset.
* @see #setMaxVirtualMemoryForTask(long)
* @deprecated Use {@link #getMemoryForMapTask()} and
* {@link #getMemoryForReduceTask()}
@@ -1760,24 +1798,16 @@
"getMaxVirtualMemoryForTask() is deprecated. " +
"Instead use getMemoryForMapTask() and getMemoryForReduceTask()");
- if (get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) == null) {
- if (get(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY) != null || get(
- JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY) != null) {
- long val = Math.max(getMemoryForMapTask(), getMemoryForReduceTask());
- if (val == JobConf.DISABLED_MEMORY_LIMIT) {
- return val;
- } else {
- if (val < 0) {
- return JobConf.DISABLED_MEMORY_LIMIT;
- }
- return val * 1024 * 1024;
- //Convert MB to byte as new value is in
- // MB and old deprecated method returns bytes
- }
+ long value = getLong(MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
+ value = normalizeMemoryConfigValue(value);
+ if (value == DISABLED_MEMORY_LIMIT) {
+ value = Math.max(getMemoryForMapTask(), getMemoryForReduceTask());
+ value = normalizeMemoryConfigValue(value);
+ if (value != DISABLED_MEMORY_LIMIT) {
+ value *= 1024*1024;
}
}
-
- return getLong(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, DISABLED_MEMORY_LIMIT);
+ return value;
}
/**
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java?rev=824273&r1=824272&r2=824273&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/conf/TestJobConf.java Mon Oct 12 09:30:49 2009
@@ -17,14 +17,15 @@
*/
package org.apache.hadoop.conf;
-import junit.framework.Assert;
-import junit.framework.TestCase;
+import org.junit.Assert;
+import org.junit.Test;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.JobContext;
-public class TestJobConf extends TestCase {
+public class TestJobConf {
+ @Test
public void testProfileParamsDefaults() {
JobConf configuration = new JobConf();
@@ -37,6 +38,7 @@
Assert.assertTrue(result.startsWith("-agentlib:hprof"));
}
+ @Test
public void testProfileParamsSetter() {
JobConf configuration = new JobConf();
@@ -44,6 +46,7 @@
Assert.assertEquals("test", configuration.get(JobContext.TASK_PROFILE_PARAMS));
}
+ @Test
public void testProfileParamsGetter() {
JobConf configuration = new JobConf();
@@ -55,6 +58,7 @@
* Testing mapred.task.maxvmem replacement with new values
*
*/
+ @Test
public void testMemoryConfigForMapOrReduceTask(){
JobConf configuration = new JobConf();
configuration.set(JobContext.MAP_MEMORY_MB,String.valueOf(300));
@@ -71,9 +75,9 @@
configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , "-1");
configuration.set(JobContext.MAP_MEMORY_MB,String.valueOf(300));
- configuration.set(JobContext.REDUCE_MEMORY_MB,String.valueOf(300));
- Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
- Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);
+ configuration.set(JobContext.REDUCE_MEMORY_MB,String.valueOf(400));
+ Assert.assertEquals(configuration.getMemoryForMapTask(), 300);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(), 400);
configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
@@ -91,13 +95,53 @@
configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
+ configuration.set(JobContext.MAP_MEMORY_MB, "3");
+ configuration.set(JobContext.REDUCE_MEMORY_MB, "3");
Assert.assertEquals(configuration.getMemoryForMapTask(),2);
Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
+
}
/**
+ * Test that negative values for MAPRED_TASK_MAXVMEM_PROPERTY cause
+ * new configuration keys' values to be used.
+ */
+ @Test
+ public void testNegativeValueForTaskVmem() {
+ JobConf configuration = new JobConf();
+
+ configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
+ configuration.set(JobContext.MAP_MEMORY_MB, "4");
+ configuration.set(JobContext.REDUCE_MEMORY_MB, "5");
+ Assert.assertEquals(4, configuration.getMemoryForMapTask());
+ Assert.assertEquals(5, configuration.getMemoryForReduceTask());
+
+ }
+
+ /**
+ * Test that negative values for all memory configuration properties causes
+ * APIs to disable memory limits
+ */
+ @Test
+ public void testNegativeValuesForMemoryParams() {
+ JobConf configuration = new JobConf();
+
+ configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-4");
+ configuration.set(JobContext.MAP_MEMORY_MB, "-5");
+ configuration.set(JobContext.REDUCE_MEMORY_MB, "-6");
+
+ Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
+ configuration.getMemoryForMapTask());
+ Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
+ configuration.getMemoryForReduceTask());
+ Assert.assertEquals(JobConf.DISABLED_MEMORY_LIMIT,
+ configuration.getMaxVirtualMemoryForTask());
+ }
+
+ /**
* Test deprecated accessor and mutator method for mapred.task.maxvmem
*/
+ @Test
public void testMaxVirtualMemoryForTask() {
JobConf configuration = new JobConf();
@@ -138,5 +182,7 @@
configuration.setMaxVirtualMemoryForTask(2 * 1024 * 1024);
Assert.assertEquals(configuration.getMemoryForMapTask(), 2);
Assert.assertEquals(configuration.getMemoryForReduceTask(), 2);
+
+
}
}