You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2013/01/09 06:31:54 UTC
svn commit: r1430690 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ bin/ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-mapreduce-c...
Author: acmurthy
Date: Wed Jan 9 05:31:53 2013
New Revision: 1430690
URL: http://svn.apache.org/viewvc?rev=1430690&view=rev
Log:
Merge -c 1430688 from trunk to branch-2 to fix MAPREDUCE-4520. Added support for MapReduce applications to request for CPU cores along-with memory post YARN-2. Contributed by Arun C. Murthy.
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-2/hadoop-mapreduce-project/bin/ (props changed)
hadoop/common/branches/branch-2/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (contents, props changed)
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples/ (props changed)
Propchange: hadoop/common/branches/branch-2/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1430688
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1430690&r1=1430689&r2=1430690&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed Jan 9 05:31:53 2013
@@ -9,6 +9,9 @@ Release 2.0.3-alpha - Unreleased
NEW FEATURES
+ MAPREDUCE-4520. Added support for MapReduce applications to request for
+ CPU cores along-with memory post YARN-2. (acmurthy)
+
IMPROVEMENTS
MAPREDUCE-3678. The Map tasks logs should have the value of input
Propchange: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1430688
Propchange: hadoop/common/branches/branch-2/hadoop-mapreduce-project/bin/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/bin:r1430688
Propchange: hadoop/common/branches/branch-2/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1430688
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1430690&r1=1430689&r2=1430690&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Wed Jan 9 05:31:53 2013
@@ -1050,6 +1050,10 @@ public class JobImpl implements org.apac
conf.getInt(MRJobConfig.MR_AM_VMEM_MB,
MRJobConfig.DEFAULT_MR_AM_VMEM_MB);
+ long sysCPUSizeForUberSlot =
+ conf.getInt(MRJobConfig.MR_AM_CPU_VCORES,
+ MRJobConfig.DEFAULT_MR_AM_CPU_VCORES);
+
boolean uberEnabled =
conf.getBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
boolean smallNumMapTasks = (numMapTasks <= sysMaxMaps);
@@ -1061,6 +1065,13 @@ public class JobImpl implements org.apac
conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0))
<= sysMemSizeForUberSlot)
|| (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT));
+ boolean smallCpu =
+ (
+ Math.max(
+ conf.getInt(MRJobConfig.MAP_CPU_VCORES, 1),
+ conf.getInt(MRJobConfig.REDUCE_CPU_VCORES, 1)) <
+ sysCPUSizeForUberSlot
+ );
boolean notChainJob = !isChainJob(conf);
// User has overall veto power over uberization, or user can modify
@@ -1071,7 +1082,8 @@ public class JobImpl implements org.apac
// while "uber-AM" (MR AM + LocalContainerLauncher) loops over tasks
// and thus requires sequential execution.
isUber = uberEnabled && smallNumMapTasks && smallNumReduceTasks
- && smallInput && smallMemory && notChainJob && isValidUberMaxReduces;
+ && smallInput && smallMemory && smallCpu
+ && notChainJob && isValidUberMaxReduces;
if (isUber) {
LOG.info("Uberizing job " + jobId + ": " + numMapTasks + "m+"
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1430690&r1=1430689&r2=1430690&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Jan 9 05:31:53 2013
@@ -527,7 +527,10 @@ public abstract class TaskAttemptImpl im
//TODO:create the resource reqt for this Task attempt
this.resourceCapability = recordFactory.newRecordInstance(Resource.class);
- this.resourceCapability.setMemory(getMemoryRequired(conf, taskId.getTaskType()));
+ this.resourceCapability.setMemory(
+ getMemoryRequired(conf, taskId.getTaskType()));
+ this.resourceCapability.setVirtualCores(
+ getCpuRequired(conf, taskId.getTaskType()));
this.dataLocalHosts = dataLocalHosts;
RackResolver.init(conf);
@@ -551,6 +554,21 @@ public abstract class TaskAttemptImpl im
return memory;
}
+ private int getCpuRequired(Configuration conf, TaskType taskType) {
+ int vcores = 1;
+ if (taskType == TaskType.MAP) {
+ vcores =
+ conf.getInt(MRJobConfig.MAP_CPU_VCORES,
+ MRJobConfig.DEFAULT_MAP_CPU_VCORES);
+ } else if (taskType == TaskType.REDUCE) {
+ vcores =
+ conf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
+ MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
+ }
+
+ return vcores;
+ }
+
/**
* Create a {@link LocalResource} record with all the given parameters.
*/
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1430690&r1=1430689&r2=1430690&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Wed Jan 9 05:31:53 2013
@@ -181,7 +181,8 @@ public interface MRJobConfig {
public static final String MAP_MEMORY_MB = "mapreduce.map.memory.mb";
public static final int DEFAULT_MAP_MEMORY_MB = 1024;
- public static final String MAP_MEMORY_PHYSICAL_MB = "mapreduce.map.memory.physical.mb";
+ public static final String MAP_CPU_VCORES = "mapreduce.map.cpu.vcores";
+ public static final int DEFAULT_MAP_CPU_VCORES = 1;
public static final String MAP_ENV = "mapreduce.map.env";
@@ -225,11 +226,12 @@ public interface MRJobConfig {
public static final String REDUCE_MARKRESET_BUFFER_SIZE = "mapreduce.reduce.markreset.buffer.size";
- public static final String REDUCE_MEMORY_PHYSICAL_MB = "mapreduce.reduce.memory.physical.mb";
-
public static final String REDUCE_MEMORY_MB = "mapreduce.reduce.memory.mb";
public static final int DEFAULT_REDUCE_MEMORY_MB = 1024;
+ public static final String REDUCE_CPU_VCORES = "mapreduce.reduce.cpu.vcores";
+ public static final int DEFAULT_REDUCE_CPU_VCORES = 1;
+
public static final String REDUCE_MEMORY_TOTAL_BYTES = "mapreduce.reduce.memory.totalbytes";
public static final String SHUFFLE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.shuffle.input.buffer.percent";
@@ -350,6 +352,11 @@ public interface MRJobConfig {
MR_AM_PREFIX+"resource.mb";
public static final int DEFAULT_MR_AM_VMEM_MB = 1536;
+ /** The number of virtual cores the MR app master needs.*/
+ public static final String MR_AM_CPU_VCORES =
+ MR_AM_PREFIX+"resource.cpu-vcores";
+ public static final int DEFAULT_MR_AM_CPU_VCORES = 1;
+
/** Command line arguments passed to the MR app master.*/
public static final String MR_AM_COMMAND_OPTS =
MR_AM_PREFIX+"command-opts";
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1430690&r1=1430689&r2=1430690&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Wed Jan 9 05:31:53 2013
@@ -475,6 +475,22 @@
</property>
<property>
+ <name>mapreduce.map.cpu.vcores</name>
+ <value>1</value>
+ <description>
+ The number of virtual cores required for each map task.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.reduce.cpu.vcores</name>
+ <value>1</value>
+ <description>
+ The number of virtual cores required for each reduce task.
+ </description>
+</property>
+
+<property>
<name>mapreduce.reduce.merge.inmem.threshold</name>
<value>1000</value>
<description>The threshold, in terms of the number of files
@@ -1341,6 +1357,14 @@
</property>
<property>
+ <name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
+ <value>1</value>
+ <description>
+ The number of virtual CPU cores the MR AppMaster needs.
+ </description>
+</property>
+
+<property>
<description>CLASSPATH for MR applications. A comma-separated list
of CLASSPATH entries</description>
<name>mapreduce.application.classpath</name>
Propchange: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1430688
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1430690&r1=1430689&r2=1430690&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Wed Jan 9 05:31:53 2013
@@ -324,8 +324,16 @@ public class YARNRunner implements Clien
// Setup resource requirements
Resource capability = recordFactory.newRecordInstance(Resource.class);
- capability.setMemory(conf.getInt(MRJobConfig.MR_AM_VMEM_MB,
- MRJobConfig.DEFAULT_MR_AM_VMEM_MB));
+ capability.setMemory(
+ conf.getInt(
+ MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
+ )
+ );
+ capability.setVirtualCores(
+ conf.getInt(
+ MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
+ )
+ );
LOG.debug("AppMaster capability = " + capability);
// Setup LocalResources
Propchange: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-examples:r1430688