You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2013/05/01 02:42:32 UTC
svn commit: r1477889 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apa...
Author: vinodkv
Date: Wed May 1 00:42:32 2013
New Revision: 1477889
URL: http://svn.apache.org/r1477889
Log:
MAPREDUCE-5145. Changed default max-attempts to be more than one for MR jobs inline with YARN. Contributed by Zhijie Shen.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1477889&r1=1477888&r2=1477889&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed May 1 00:42:32 2013
@@ -215,6 +215,9 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-5069. add concrete common implementations of
CombineFileInputFormat (Sangjin Lee via bobby)
+ MAPREDUCE-5145. Changed default max-attempts to be more than one for MR jobs
+ inline with YARN. (Zhijie Shen via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java?rev=1477889&r1=1477888&r2=1477889&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java Wed May 1 00:42:32 2013
@@ -90,6 +90,7 @@ import org.junit.Test;
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
+ Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf);
@@ -116,8 +117,9 @@ import org.junit.Test;
appId.setId(0);
attemptId.setApplicationId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
+ Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
- JobStateInternal.REBOOT, 4);
+ JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
appMaster.init(conf);
appMaster.start();
//shutdown the job, not the lastRetry
@@ -144,7 +146,7 @@ import org.junit.Test;
attemptId.setApplicationId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
- JobStateInternal.REBOOT, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
+ JobStateInternal.REBOOT, 1); //no retry
appMaster.init(conf);
appMaster.start();
//shutdown the job, is lastRetry
@@ -201,8 +203,7 @@ import org.junit.Test;
JobId jobid = recordFactory.newRecordInstance(JobId.class);
jobid.setAppId(appId);
ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
- MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
- MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
+ MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, 1); //no retry
appMaster.init(conf);
//simulate the process being killed
MRAppMaster.MRAppMasterShutdownHook hook =
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1477889&r1=1477888&r2=1477889&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Wed May 1 00:42:32 2013
@@ -677,6 +677,6 @@ public interface MRJobConfig {
*/
public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts";
- public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 1;
+ public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 2;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1477889&r1=1477888&r2=1477889&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Wed May 1 00:42:32 2013
@@ -816,10 +816,11 @@
<property>
<name>mapreduce.am.max-attempts</name>
- <value>1</value>
+ <value>2</value>
<description>The maximum number of application attempts. It is a
application-specific setting. It should not be larger than the global number
- set by resourcemanager. Otherwise, it will be override.</description>
+ set by resourcemanager. Otherwise, it will be override. The default number is
+ set to 2, to allow at least one retry for AM.</description>
</property>
<!-- Job Notification Configuration -->