You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2013/06/04 01:57:09 UTC
svn commit: r1489237 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/...
Author: acmurthy
Date: Mon Jun 3 23:57:08 2013
New Revision: 1489237
URL: http://svn.apache.org/r1489237
Log:
Merge -c 1489236 from trunk to branch-2 to fix MAPREDUCE-5245. Added back constants to JobConf to fix incompatibilities. Contributed by Zhijie Shen.
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1489237&r1=1489236&r2=1489237&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Mon Jun 3 23:57:08 2013
@@ -353,6 +353,9 @@ Release 2.1.0-beta - UNRELEASED
MAPREDUCE-5296. Fix backward incompatibility for JobControl by adding the
omitted addJob. (Zhijie Shen via acmurthy)
+ MAPREDUCE-5245. Added back constants to JobConf to fix incompatibilities.
+ (Zhijie Shen via acmurthy)
+
BREAKDOWN OF HADOOP-8562 SUBTASKS
MAPREDUCE-4739. Some MapReduce tests fail to find winutils.
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1489237&r1=1489236&r2=1489237&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java Mon Jun 3 23:57:08 2013
@@ -20,9 +20,6 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.Enumeration;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
@@ -122,8 +119,8 @@ public class JobConf extends Configurati
}
/**
- * @deprecated Use {@link #MAPRED_JOB_MAP_MEMORY_MB_PROPERTY} and
- * {@link #MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY}
+ * @deprecated Use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY} and
+ * {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}
*/
@Deprecated
public static final String MAPRED_TASK_MAXVMEM_PROPERTY =
@@ -167,12 +164,28 @@ public class JobConf extends Configurati
*/
public static final String DEFAULT_QUEUE_NAME = "default";
- static final String MAPRED_JOB_MAP_MEMORY_MB_PROPERTY =
+ static final String MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY =
JobContext.MAP_MEMORY_MB;
- static final String MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY =
+ static final String MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY =
JobContext.REDUCE_MEMORY_MB;
+ /**
+ * The variable is kept for M/R 1.x applications, while M/R 2.x applications
+ * should use {@link #MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY}
+ */
+ @Deprecated
+ public static final String MAPRED_JOB_MAP_MEMORY_MB_PROPERTY =
+ "mapred.job.map.memory.mb";
+
+ /**
+ * The variable is kept for M/R 1.x applications, while M/R 2.x applications
+ * should use {@link #MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY}
+ */
+ @Deprecated
+ public static final String MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY =
+ "mapred.job.reduce.memory.mb";
+
/** Pattern for the default unpacking behavior for job jars */
public static final Pattern UNPACK_JAR_PATTERN_DEFAULT =
Pattern.compile("(?:classes/|lib/).*");
@@ -335,8 +348,67 @@ public class JobConf extends Configurati
* Default logging level for map/reduce tasks.
*/
public static final Level DEFAULT_LOG_LEVEL = Level.INFO;
-
-
+
+ /**
+ * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ * use {@link MRJobConfig#WORKFLOW_ID} instead
+ */
+ @Deprecated
+ public static final String WORKFLOW_ID = MRJobConfig.WORKFLOW_ID;
+
+ /**
+ * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ * use {@link MRJobConfig#WORKFLOW_NAME} instead
+ */
+ @Deprecated
+ public static final String WORKFLOW_NAME = MRJobConfig.WORKFLOW_NAME;
+
+ /**
+ * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ * use {@link MRJobConfig#WORKFLOW_NODE_NAME} instead
+ */
+ @Deprecated
+ public static final String WORKFLOW_NODE_NAME =
+ MRJobConfig.WORKFLOW_NODE_NAME;
+
+ /**
+ * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ * use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_STRING} instead
+ */
+ @Deprecated
+ public static final String WORKFLOW_ADJACENCY_PREFIX_STRING =
+ MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_STRING;
+
+ /**
+ * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ * use {@link MRJobConfig#WORKFLOW_ADJACENCY_PREFIX_PATTERN} instead
+ */
+ @Deprecated
+ public static final String WORKFLOW_ADJACENCY_PREFIX_PATTERN =
+ MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_PATTERN;
+
+ /**
+ * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ * use {@link MRJobConfig#WORKFLOW_TAGS} instead
+ */
+ @Deprecated
+ public static final String WORKFLOW_TAGS = MRJobConfig.WORKFLOW_TAGS;
+
+ /**
+ * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ * not use it
+ */
+ @Deprecated
+ public static final String MAPREDUCE_RECOVER_JOB =
+ "mapreduce.job.restart.recover";
+
+ /**
+ * The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ * not use it
+ */
+ @Deprecated
+ public static final boolean DEFAULT_MAPREDUCE_RECOVER_JOB = true;
+
/**
* Construct a map/reduce job configuration.
*/
@@ -1697,6 +1769,12 @@ public class JobConf extends Configurati
long value = getDeprecatedMemoryValue();
if (value == DISABLED_MEMORY_LIMIT) {
value = normalizeMemoryConfigValue(
+ getLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY,
+ DISABLED_MEMORY_LIMIT));
+ }
+ // In case that M/R 1.x applications use the old property name
+ if (value == DISABLED_MEMORY_LIMIT) {
+ value = normalizeMemoryConfigValue(
getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
DISABLED_MEMORY_LIMIT));
}
@@ -1704,6 +1782,8 @@ public class JobConf extends Configurati
}
public void setMemoryForMapTask(long mem) {
+ setLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, mem);
+ // In case that M/R 1.x applications use the old property name
setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, mem);
}
@@ -1724,6 +1804,12 @@ public class JobConf extends Configurati
long value = getDeprecatedMemoryValue();
if (value == DISABLED_MEMORY_LIMIT) {
value = normalizeMemoryConfigValue(
+ getLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY,
+ DISABLED_MEMORY_LIMIT));
+ }
+ // In case that M/R 1.x applications use the old property name
+ if (value == DISABLED_MEMORY_LIMIT) {
+ value = normalizeMemoryConfigValue(
getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
DISABLED_MEMORY_LIMIT));
}
@@ -1745,6 +1831,8 @@ public class JobConf extends Configurati
}
public void setMemoryForReduceTask(long mem) {
+ setLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, mem);
+ // In case that M/R 1.x applications use the old property name
setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, mem);
}
@@ -1927,8 +2015,8 @@ public class JobConf extends Configurati
private void checkAndWarnDeprecation() {
if(get(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY) != null) {
LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY)
- + " Instead use " + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY
- + " and " + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
+ + " Instead use " + JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY
+ + " and " + JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY);
}
if(get(JobConf.MAPRED_TASK_ULIMIT) != null ) {
LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_ULIMIT));
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java?rev=1489237&r1=1489236&r2=1489237&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java Mon Jun 3 23:57:08 2013
@@ -22,6 +22,7 @@ import java.util.regex.Pattern;
import static org.junit.Assert.*;
import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
import org.junit.Test;
/**
@@ -152,4 +153,36 @@ public class TestJobConf {
JobConf.deprecatedString("key"));
}
+
+ /**
+ * Ensure that M/R 1.x applications can get and set task virtual memory with
+ * old property names
+ */
+ @SuppressWarnings("deprecation")
+ @Test (timeout = 1000)
+ public void testDeprecatedPropertyNameForTaskVmem() {
+ JobConf configuration = new JobConf();
+
+ configuration.setLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, 1024);
+ configuration.setLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, 1024);
+ Assert.assertEquals(1024, configuration.getMemoryForMapTask());
+ Assert.assertEquals(1024, configuration.getMemoryForReduceTask());
+ // Make sure new property names aren't broken by the old ones
+ configuration.setLong(JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, 1025);
+ configuration.setLong(JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, 1025);
+ Assert.assertEquals(1025, configuration.getMemoryForMapTask());
+ Assert.assertEquals(1025, configuration.getMemoryForReduceTask());
+
+ configuration.setMemoryForMapTask(2048);
+ configuration.setMemoryForReduceTask(2048);
+ Assert.assertEquals(2048, configuration.getLong(
+ JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY, -1));
+ Assert.assertEquals(2048, configuration.getLong(
+ JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY, -1));
+ // Make sure new property names aren't broken by the old ones
+ Assert.assertEquals(2048, configuration.getLong(
+ JobConf.MAPREDUCE_JOB_MAP_MEMORY_MB_PROPERTY, -1));
+ Assert.assertEquals(2048, configuration.getLong(
+ JobConf.MAPREDUCE_JOB_REDUCE_MEMORY_MB_PROPERTY, -1));
+ }
}