You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/10/13 22:34:56 UTC
[06/51] [abbrv] hadoop git commit: MAPREDUCE-6776.
yarn.app.mapreduce.client.job.max-retries should have a more useful default
(miklos.szegedi@cloudera.com via rkanter)
MAPREDUCE-6776. yarn.app.mapreduce.client.job.max-retries should have a more useful default (miklos.szegedi@cloudera.com via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3f37e6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3f37e6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3f37e6f
Branch: refs/heads/HDFS-7240
Commit: f3f37e6fb8172f6434e06eb9a137c0c155b3952e
Parents: 2e853be
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Oct 7 14:47:06 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Fri Oct 7 14:47:06 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/MRJobConfig.java | 2 +-
.../src/main/resources/mapred-default.xml | 10 +++---
.../apache/hadoop/mapred/JobClientUnitTest.java | 34 ++++++++++++++++----
3 files changed, 34 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f37e6f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 5716404..1325b74 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -505,7 +505,7 @@ public interface MRJobConfig {
*/
public static final String MR_CLIENT_JOB_MAX_RETRIES =
MR_PREFIX + "client.job.max-retries";
- public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 0;
+ public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 3;
/**
* How long to wait between jobclient retries on failure
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f37e6f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 73aaa7a..fe29212 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1505,12 +1505,12 @@
<property>
<name>yarn.app.mapreduce.client.job.max-retries</name>
- <value>0</value>
+ <value>3</value>
<description>The number of retries the client will make for getJob and
- dependent calls. The default is 0 as this is generally only needed for
- non-HDFS DFS where additional, high level retries are required to avoid
- spurious failures during the getJob call. 30 is a good value for
- WASB</description>
+ dependent calls.
+ This is needed for non-HDFS DFS where additional, high level
+ retries are required to avoid spurious failures during the getJob call.
+ 30 is a good value for WASB</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3f37e6f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
index 4895a5b..e02232d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobClientUnitTest.java
@@ -225,10 +225,10 @@ public class JobClientUnitTest {
//To prevent the test from running for a very long time, lower the retry
JobConf conf = new JobConf();
- conf.set(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, "3");
+ conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, 2);
TestJobClientGetJob client = new TestJobClientGetJob(conf);
- JobID id = new JobID("ajob",1);
+ JobID id = new JobID("ajob", 1);
RunningJob rj = mock(RunningJob.class);
client.setRunningJob(rj);
@@ -236,13 +236,35 @@ public class JobClientUnitTest {
assertNotNull(client.getJob(id));
assertEquals(client.getLastGetJobRetriesCounter(), 0);
- //3 retry
+ //2 retries
+ client.setGetJobRetries(2);
+ assertNotNull(client.getJob(id));
+ assertEquals(client.getLastGetJobRetriesCounter(), 2);
+
+ //beyond yarn.app.mapreduce.client.job.max-retries, will get null
client.setGetJobRetries(3);
+ assertNull(client.getJob(id));
+ }
+
+ @Test
+ public void testGetJobRetryDefault() throws Exception {
+
+ //To prevent the test from running for a very long time, lower the retry
+ JobConf conf = new JobConf();
+
+ TestJobClientGetJob client = new TestJobClientGetJob(conf);
+ JobID id = new JobID("ajob", 1);
+ RunningJob rj = mock(RunningJob.class);
+ client.setRunningJob(rj);
+
+ //3 retries (default)
+ client.setGetJobRetries(MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES);
assertNotNull(client.getJob(id));
- assertEquals(client.getLastGetJobRetriesCounter(), 3);
+ assertEquals(client.getLastGetJobRetriesCounter(),
+ MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES);
- //beyond MAPREDUCE_JOBCLIENT_GETJOB_MAX_RETRY_KEY, will get null
- client.setGetJobRetries(5);
+ //beyond yarn.app.mapreduce.client.job.max-retries, will get null
+ client.setGetJobRetries(MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES + 1);
assertNull(client.getJob(id));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org