You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2008/09/29 12:59:52 UTC
svn commit: r700056 - in /hadoop/core/trunk: CHANGES.txt
src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
Author: ddas
Date: Mon Sep 29 03:59:51 2008
New Revision: 700056
URL: http://svn.apache.org/viewvc?rev=700056&view=rev
Log:
HADOOP-4237. Fixes the TestStreamingBadRecords.testNarrowDown testcase. Contributed by Sharad Agarwal.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=700056&r1=700055&r2=700056&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Sep 29 03:59:51 2008
@@ -778,6 +778,9 @@
HADOOP-4190. Fixes the backward compatibility issue with Job History.
introduced by HADOOP-3245 and HADOOP-2403. (Amar Kamat via ddas)
+ HADOOP-4237. Fixes the TestStreamingBadRecords.testNarrowDown testcase.
+ (Sharad Agarwal via ddas)
+
Release 0.18.2 - Unreleased
BUG FIXES
Modified: hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java?rev=700056&r1=700055&r2=700056&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java (original)
+++ hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java Mon Sep 29 03:59:51 2008
@@ -170,7 +170,6 @@
"-jobconf", "mapred.skip.reduce.max.skip.groups="+Long.MAX_VALUE,
"-jobconf", "mapred.map.tasks=1",
"-jobconf", "mapred.reduce.tasks=1",
- "-jobconf", "mapred.task.timeout=30000",
"-jobconf", "fs.default.name="+clusterConf.get("fs.default.name"),
"-jobconf", "mapred.job.tracker="+clusterConf.get("mapred.job.tracker"),
"-jobconf", "mapred.job.tracker.http.address="
@@ -197,13 +196,15 @@
"-verbose",
"-inputformat", "org.apache.hadoop.mapred.KeyValueTextInputFormat",
"-jobconf", "mapred.skip.attempts.to.start.skipping=1",
- "-jobconf", "mapred.map.max.attempts=12",
- "-jobconf", "mapred.reduce.max.attempts=8",
+ //actually fewer attempts are required than specified
+ //but to cater to the case of slow processed counter update, need to
+ //have more attempts
+ "-jobconf", "mapred.map.max.attempts=20",
+ "-jobconf", "mapred.reduce.max.attempts=15",
"-jobconf", "mapred.skip.map.max.skip.records=1",
"-jobconf", "mapred.skip.reduce.max.skip.groups=1",
"-jobconf", "mapred.map.tasks=1",
"-jobconf", "mapred.reduce.tasks=1",
- "-jobconf", "mapred.task.timeout=30000",
"-jobconf", "fs.default.name="+clusterConf.get("fs.default.name"),
"-jobconf", "mapred.job.tracker="+clusterConf.get("mapred.job.tracker"),
"-jobconf", "mapred.job.tracker.http.address="
@@ -275,7 +276,7 @@
}
else if(badRecords.size()>2 && line.contains(badRecords.get(2))) {
LOG.warn("Encountered BAD record");
- Thread.sleep(15*60*1000);
+ System.exit(-1);
}
super.processLine(line);
}