You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:39:39 UTC
svn commit: r1077643 - in
/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred:
JobInProgress.java ResourceEstimator.java
Author: omalley
Date: Fri Mar 4 04:39:39 2011
New Revision: 1077643
URL: http://svn.apache.org/viewvc?rev=1077643&view=rev
Log:
commit 6ae4bf9e0e092235ef0f5d9bd13ac24a6e6c75a7
Author: Arun C Murthy <ac...@apache.org>
Date: Thu Aug 5 23:54:58 2010 -0700
MAPREDUCE-1521. Fixed estimation of map outputs to be consistently use the same number of maps as reduce slow-start to ensure reduces aren't launched too early. Contributed by Mahadev Konar.
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1077643&r1=1077642&r2=1077643&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java Fri Mar 4 04:39:39 2011
@@ -722,7 +722,10 @@ public class JobInProgress {
(conf.getFloat("mapred.reduce.slowstart.completed.maps",
DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART) *
numMapTasks));
-
+
+ // ... use the same for estimating the total output of all maps
+ resourceEstimator.setThreshhold(completedMapsForReduceSlowstart);
+
// create cleanup two cleanup tips, one map and one reduce.
cleanup = new TaskInProgress[2];
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java?rev=1077643&r1=1077642&r2=1077643&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ResourceEstimator.java Fri Mar 4 04:39:39 2011
@@ -39,7 +39,7 @@ class ResourceEstimator {
private int completedMapsUpdates;
final private JobInProgress job;
- final private int threshholdToUse;
+ private int threshholdToUse;
public ResourceEstimator(JobInProgress job) {
this.job = job;
@@ -106,5 +106,14 @@ class ResourceEstimator {
}
}
-
+ /**
+ * the number of maps after which reduce starts launching
+ * @param numMaps the number of maps after which reduce starts
+ * launching. It acts as the upper bound for the threshhold, so
+ * that we can get right estimates before we reach these number
+ * of maps.
+ */
+ void setThreshhold(int numMaps) {
+ threshholdToUse = Math.min(threshholdToUse, numMaps);
+ }
}