You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/06/20 07:51:39 UTC
svn commit: r1137506 - in /hadoop/common/branches/MR-279/mapreduce:
CHANGES.txt
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Author: vinodkv
Date: Mon Jun 20 05:51:38 2011
New Revision: 1137506
URL: http://svn.apache.org/viewvc?rev=1137506&view=rev
Log:
Fix to schedule reduces irrespective of the headroom when all maps are done so as to avoid stall in reduce-scheduling when slow-start is disabled. Contributed by Sharad Agarwal.
Modified:
hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Modified: hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/CHANGES.txt?rev=1137506&r1=1137505&r2=1137506&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/MR-279/mapreduce/CHANGES.txt Mon Jun 20 05:51:38 2011
@@ -5,6 +5,10 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Fix to schedule reduces irrespective of the headroom when all maps are
+ done so as to avoid stall in reduce-scheduling when slow-start is
+ disabled. (Sharad Agarwal via vinodkv).
+
Fix RM app start/finish time and diagnostics. (llu)
Fix race condition between multiple localizers on a single node. (cdouglas via mahadev)
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1137506&r1=1137505&r2=1137506&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Mon Jun 20 05:51:38 2011
@@ -258,14 +258,23 @@ public class RMContainerAllocator extend
}
//check if reduces have taken over the whole cluster and there are
//unassigned maps
- int memLimit = getMemLimit();
if (scheduledRequests.maps.size() > 0) {
+ int memLimit = getMemLimit();
int availableMemForMap = memLimit - ((assignedRequests.reduces.size() -
assignedRequests.preemptionWaitingReduces.size()) * reduceResourceReqt);
//availableMemForMap must be sufficient to run atleast 1 map
if (availableMemForMap < mapResourceReqt) {
+ //to make sure new containers are given to maps and not reduces
+ //ramp down all scheduled reduces if any
+ //(since reduces are scheduled at higher priority than maps)
+ LOG.info("Ramping down all scheduled reduces:" + scheduledRequests.reduces.size());
+ for (ContainerRequest req : scheduledRequests.reduces.values()) {
+ pendingReduces.add(req);
+ }
+ scheduledRequests.reduces.clear();
+
//preempt for making space for atleast one map
- int premeptionLimit = Math.max(mapResourceReqt - availableMemForMap,
+ int premeptionLimit = Math.max(mapResourceReqt,
(int) (maxReducePreemptionLimit * memLimit));
int preemptMem = Math.min(scheduledRequests.maps.size() * mapResourceReqt,
@@ -288,6 +297,18 @@ public class RMContainerAllocator extend
LOG.info("Recalculating schedule...");
+ //if all maps are assigned, then ramp up all reduces irrespective of the
+ //headroom
+ if (scheduledRequests.maps.size() == 0 && pendingReduces.size() > 0) {
+ LOG.info("All maps assigned. Ramping up all remaining reduces:" + pendingReduces.size());
+ for (ContainerRequest req : pendingReduces) {
+ scheduledRequests.addReduce(req);
+ }
+ pendingReduces.clear();
+ return;
+ }
+
+
int totalMaps = assignedRequests.maps.size() + completedMaps + scheduledRequests.maps.size();
//check for slow start