You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/09/02 05:51:30 UTC
svn commit: r691099 - in /hadoop/core/trunk: ./ conf/
src/contrib/streaming/src/test/org/apache/hadoop/streaming/
src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/
Author: cdouglas
Date: Mon Sep 1 20:51:29 2008
New Revision: 691099
URL: http://svn.apache.org/viewvc?rev=691099&view=rev
Log:
HADOOP-3954. Disable record skipping by default. Contributed by Sharad Agarwal.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/conf/hadoop-default.xml
hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SkipBadRecords.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestBadRecords.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=691099&r1=691098&r2=691099&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Sep 1 20:51:29 2008
@@ -440,6 +440,9 @@
HADOOP-3910. Remove unused ClusterTestDFSNamespaceLogging and
ClusterTestDFS. (Tsz Wo (Nicholas), SZE via cdouglas)
+ HADOOP-3954. Disable record skipping by default. (Sharad Agarwal via
+ cdouglas)
+
Release 0.18.1 - Unreleased
BUG FIXES
Modified: hadoop/core/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/hadoop-default.xml?rev=691099&r1=691098&r2=691099&view=diff
==============================================================================
--- hadoop/core/trunk/conf/hadoop-default.xml (original)
+++ hadoop/core/trunk/conf/hadoop-default.xml Mon Sep 1 20:51:29 2008
@@ -1129,6 +1129,51 @@
<description> Number of lines per split in NLineInputFormat.
</description>
</property>
+
+ <property>
+ <name>mapred.skip.mode.enabled</name>
+ <value>false</value>
+ <description> Indicates whether skipping of bad records is enabled or not.
+ If enabled the framework will try to find bad records and skip
+ them on further attempts.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.skip.attempts.to.start.skipping</name>
+ <value>2</value>
+ <description> The number of Task attempts AFTER which skip mode
+ will be kicked off. When skip mode is kicked off, the
+ tasks reports the range of records which it will process
+ next, to the TaskTracker. So that on failures, TT knows which
+ ones are possibly the bad records. On further executions,
+ those are skipped.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.skip.map.auto.incr.proc.count</name>
+ <value>true</value>
+ <description> The flag which if set to true,
+ Counters.Application.MAP_PROCESSED_RECORDS is incremented
+ by MapRunner after invoking the map function. This value must be set to
+ false for applications which process the records asynchronously
+ or buffer the input records. For example streaming.
+ In such cases applications should increment this counter on their own.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.skip.reduce.auto.incr.proc.count</name>
+ <value>true</value>
+ <description> The flag which if set to true,
+ Counters.Application.REDUCE_PROCESSED_RECORDS is incremented
+ by framework after invoking the reduce function. This value must be set to
+ false for applications which process the records asynchronously
+ or buffer the input records. For example streaming.
+ In such cases applications should increment this counter on their own.
+ </description>
+ </property>
<!-- ipc properties -->
Modified: hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java?rev=691099&r1=691098&r2=691099&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java (original)
+++ hadoop/core/trunk/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java Mon Sep 1 20:51:29 2008
@@ -174,6 +174,7 @@
"-jobconf", "mapred.skip.attempts.to.start.skipping="+attSkip,
"-jobconf", "mapred.map.max.attempts="+mapperAttempts,
"-jobconf", "mapred.reduce.max.attempts="+reducerAttempts,
+ "-jobconf", "mapred.skip.mode.enabled=true",
"-jobconf", "mapred.map.tasks=1",
"-jobconf", "mapred.reduce.tasks=1",
"-jobconf", "mapred.task.timeout=30000",
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SkipBadRecords.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SkipBadRecords.java?rev=691099&r1=691098&r2=691099&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SkipBadRecords.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/SkipBadRecords.java Mon Sep 1 20:51:29 2008
@@ -46,7 +46,7 @@
* <code>false</code> otherwise.
*/
public static boolean getEnabled(Configuration conf) {
- return conf.getBoolean(ENABLED, true);
+ return conf.getBoolean(ENABLED, false);
}
/**
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestBadRecords.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestBadRecords.java?rev=691099&r1=691098&r2=691099&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestBadRecords.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestBadRecords.java Mon Sep 1 20:51:29 2008
@@ -72,6 +72,7 @@
conf.setNumMapTasks(1);
conf.setNumReduceTasks(1);
conf.setInt("mapred.task.timeout", 30*1000);
+ SkipBadRecords.setEnabled(conf, true);
SkipBadRecords.setAttemptsToStartSkipping(conf,0);
//the no of attempts to successfully complete the task depends