You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2014/07/07 22:14:33 UTC
svn commit: r1608596 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/s...
Author: vinodkv
Date: Mon Jul 7 20:14:33 2014
New Revision: 1608596
URL: http://svn.apache.org/r1608596
Log:
MAPREDUCE-5517. Fixed MapReduce ApplicationMaster to not validate reduce side resource configuration for deciding uber-mode on map-only jobs. Contributed by Siqi Li.
svn merge --ignore-ancestry -c 1608595 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1608596&r1=1608595&r2=1608596&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Mon Jul 7 20:14:33 2014
@@ -152,6 +152,10 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5868. Fixed an issue with TestPipeApplication that was causing the
nightly builds to fail. (Akira Ajisaka via vinodkv)
+ MAPREDUCE-5517. Fixed MapReduce ApplicationMaster to not validate reduce side
+ resource configuration for deciding uber-mode on map-only jobs. (Siqi Li via
+ vinodkv)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1608596&r1=1608595&r2=1608596&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Mon Jul 7 20:14:33 2014
@@ -1218,22 +1218,25 @@ public class JobImpl implements org.apac
boolean smallNumReduceTasks = (numReduceTasks <= sysMaxReduces);
boolean smallInput = (dataInputLength <= sysMaxBytes);
// ignoring overhead due to UberAM and statics as negligible here:
+ long requiredMapMB = conf.getLong(MRJobConfig.MAP_MEMORY_MB, 0);
+ long requiredReduceMB = conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0);
+ long requiredMB = Math.max(requiredMapMB, requiredReduceMB);
+ int requiredMapCores = conf.getInt(
+ MRJobConfig.MAP_CPU_VCORES,
+ MRJobConfig.DEFAULT_MAP_CPU_VCORES);
+ int requiredReduceCores = conf.getInt(
+ MRJobConfig.REDUCE_CPU_VCORES,
+ MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
+ int requiredCores = Math.max(requiredMapCores, requiredReduceCores);
+ if (numReduceTasks == 0) {
+ requiredMB = requiredMapMB;
+ requiredCores = requiredMapCores;
+ }
boolean smallMemory =
- ( (Math.max(conf.getLong(MRJobConfig.MAP_MEMORY_MB, 0),
- conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0))
- <= sysMemSizeForUberSlot)
- || (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT));
- boolean smallCpu =
- (
- Math.max(
- conf.getInt(
- MRJobConfig.MAP_CPU_VCORES,
- MRJobConfig.DEFAULT_MAP_CPU_VCORES),
- conf.getInt(
- MRJobConfig.REDUCE_CPU_VCORES,
- MRJobConfig.DEFAULT_REDUCE_CPU_VCORES))
- <= sysCPUSizeForUberSlot
- );
+ (requiredMB <= sysMemSizeForUberSlot)
+ || (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT);
+
+ boolean smallCpu = requiredCores <= sysCPUSizeForUberSlot;
boolean notChainJob = !isChainJob(conf);
// User has overall veto power over uberization, or user can modify
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java?rev=1608596&r1=1608595&r2=1608596&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java Mon Jul 7 20:14:33 2014
@@ -657,6 +657,15 @@ public class TestJobImpl {
conf.setInt(MRJobConfig.JOB_UBERTASK_MAXMAPS, 1);
isUber = testUberDecision(conf);
Assert.assertFalse(isUber);
+
+ // enable uber mode of 0 reducer no matter how much memory assigned to reducer
+ conf = new Configuration();
+ conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true);
+ conf.setInt(MRJobConfig.NUM_REDUCES, 0);
+ conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 2048);
+ conf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 10);
+ isUber = testUberDecision(conf);
+ Assert.assertTrue(isUber);
}
private boolean testUberDecision(Configuration conf) {