You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/06/10 15:24:30 UTC
svn commit: r1134315 - in /hadoop/mapreduce/branches/MR-279: CHANGES.txt
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Author: vinodkv
Date: Fri Jun 10 13:24:29 2011
New Revision: 1134315
URL: http://svn.apache.org/viewvc?rev=1134315&view=rev
Log:
Fix more rounding off problems in reduce ramp up. Also fix a bug preventing the application of the cap on reduce ramp-up. (Sharad Agarwal via vinodkv)
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1134315&r1=1134314&r2=1134315&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri Jun 10 13:24:29 2011
@@ -4,6 +4,10 @@ Trunk (unreleased changes)
MAPREDUCE-279
+
+ Fix more rounding off problems in reduce ramp up. Also fix a bug preventing
+ the application of the cap on reduce ramp-up. (Sharad Agarwal via vinodkv)
+
Fix rounding off problem in reduce ramp up. (sharad)
MAPREDUCE-2582. Cleanup JobHistory event generation.(Siddharth Seth via sharad)
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1134315&r1=1134314&r2=1134315&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Fri Jun 10 13:24:29 2011
@@ -249,7 +249,7 @@ public class RMContainerAllocator extend
if (availableMemForMap < mapResourceReqt) {
//preempt for making space for atleast one map
int premeptionLimit = Math.max(mapResourceReqt - availableMemForMap,
- (int) maxReducePreemptionLimit * memLimit);
+ (int) (maxReducePreemptionLimit * memLimit));
int preemptMem = Math.min(scheduledRequests.maps.size() * mapResourceReqt,
premeptionLimit);
@@ -294,34 +294,42 @@ public class RMContainerAllocator extend
completedMapPercent = 1;
}
- int scheduledMapMem = scheduledRequests.maps.size() * mapResourceReqt;
-
- int mapMemLimit = 0;
- int reduceMemLimit = 0;
+ int netScheduledMapMem = scheduledRequests.maps.size() * mapResourceReqt
+ + assignedRequests.maps.size() * mapResourceReqt;
+
+ int netScheduledReduceMem = scheduledRequests.reduces.size()
+ * reduceResourceReqt + assignedRequests.reduces.size()
+ * reduceResourceReqt;
+
+ int finalMapMemLimit = 0;
+ int finalReduceMemLimit = 0;
// ramp up the reduces based on completed map percentage
- int memLimit = getMemLimit();
- reduceMemLimit = Math.min((int)(completedMapPercent * memLimit),
- (int) maxReduceRampupLimit * memLimit);
- mapMemLimit = memLimit - reduceMemLimit;
+ int totalMemLimit = getMemLimit();
+ int idealReduceMemLimit = Math.min((int)(completedMapPercent * totalMemLimit),
+ (int) (maxReduceRampupLimit * totalMemLimit));
+ int idealMapMemLimit = totalMemLimit - idealReduceMemLimit;
// check if there aren't enough maps scheduled, give the free map capacity
// to reduce
- if (mapMemLimit > scheduledMapMem) {
- int unusedMapMemLimit = mapMemLimit - scheduledMapMem;
- reduceMemLimit = reduceMemLimit + unusedMapMemLimit;
+ if (idealMapMemLimit > netScheduledMapMem) {
+ int unusedMapMemLimit = idealMapMemLimit - netScheduledMapMem;
+ finalReduceMemLimit = idealReduceMemLimit + unusedMapMemLimit;
+ finalMapMemLimit = totalMemLimit - finalReduceMemLimit;
+ } else {
+ finalMapMemLimit = idealMapMemLimit;
+ finalReduceMemLimit = idealReduceMemLimit;
}
- int netReducesMemScheduled = scheduledRequests.reduces.size() * reduceResourceReqt
- + assignedRequests.reduces.size() * reduceResourceReqt;
-
- LOG.info("completedMapPercent " + completedMapPercent +
- " mapMemLimit:" + mapMemLimit +
- " reduceMemLimit:" + reduceMemLimit +
- " scheduledMapMem:" + scheduledMapMem +
- " netReducesScheduled:" + netReducesMemScheduled);
+ LOG.info("completedMapPercent " + completedMapPercent +
+ " totalMemLimit:" + totalMemLimit +
+ " finalMapMemLimit:" + finalMapMemLimit +
+ " finalReduceMemLimit:" + finalReduceMemLimit +
+ " netScheduledMapMem:" + netScheduledMapMem +
+ " netScheduledReduceMem:" + netScheduledReduceMem);
- int rampUp = (reduceMemLimit - netReducesMemScheduled)/reduceResourceReqt;
+ int rampUp = (finalReduceMemLimit - netScheduledReduceMem)
+ / reduceResourceReqt;
if (rampUp > 0) {
rampUp = Math.min(rampUp, pendingReduces.size());