You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by je...@apache.org on 2012/08/31 23:32:15 UTC
svn commit: r1379625 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resour...
Author: jeagles
Date: Fri Aug 31 21:32:14 2012
New Revision: 1379625
URL: http://svn.apache.org/viewvc?rev=1379625&view=rev
Log:
svn merge -c 1379617 from trunk FIXES: MAPREDUCE-4604. In mapred-default, mapreduce.map.maxattempts & mapreduce.reduce.maxattempts defaults are set to 4 as well as mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/mapred-default.xml
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1379625&r1=1379624&r2=1379625&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Aug 31 21:32:14 2012
@@ -429,6 +429,10 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
Evans via tgraves)
+ MAPREDUCE-4604. In mapred-default, mapreduce.map.maxattempts &
+ mapreduce.reduce.maxattempts defaults are set to 4 as well as
+ mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1379625&r1=1379624&r2=1379625&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java Fri Aug 31 21:32:14 2012
@@ -1357,7 +1357,7 @@ public class JobConf extends Configurati
* @return the maximum no. of failures of a given job per tasktracker.
*/
public int getMaxTaskFailuresPerTracker() {
- return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 4);
+ return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 3);
}
/**
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1379625&r1=1379624&r2=1379625&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Fri Aug 31 21:32:14 2012
@@ -782,9 +782,12 @@
<property>
<name>mapreduce.job.maxtaskfailures.per.tracker</name>
- <value>4</value>
+ <value>3</value>
<description>The number of task-failures on a tasktracker of a given job
- after which new tasks of that job aren't assigned to it.
+ after which new tasks of that job aren't assigned to it. It
+ MUST be less than mapreduce.map.maxattempts and
+ mapreduce.reduce.maxattempts otherwise the failed task will
+ never be tried on a different node.
</description>
</property>
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java?rev=1379625&r1=1379624&r2=1379625&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java Fri Aug 31 21:32:14 2012
@@ -21,6 +21,7 @@ import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig;
public class TestJobConf {
@@ -185,4 +186,19 @@ public class TestJobConf {
}
+
+ /**
+ * Ensure that by default JobContext.MAX_TASK_FAILURES_PER_TRACKER is less
+ * JobContext.MAP_MAX_ATTEMPTS and JobContext.REDUCE_MAX_ATTEMPTS so that
+ * failed tasks will be retried on other nodes
+ */
+ @Test
+ public void testMaxTaskFailuresPerTracker() {
+ JobConf jobConf = new JobConf(true);
+ Assert.assertTrue("By default JobContext.MAX_TASK_FAILURES_PER_TRACKER was "
+ + "not less than JobContext.MAP_MAX_ATTEMPTS and REDUCE_MAX_ATTEMPTS"
+ ,jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxMapAttempts() &&
+ jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts()
+ );
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/mapred-default.xml?rev=1379625&r1=1379624&r2=1379625&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/mapred-default.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/java/mapred-default.xml Fri Aug 31 21:32:14 2012
@@ -701,7 +701,7 @@
<property>
<name>mapreduce.job.maxtaskfailures.per.tracker</name>
- <value>4</value>
+ <value>3</value>
<description>The number of task-failures on a tasktracker of a given job
after which new tasks of that job aren't assigned to it.
</description>