svn commit: r1592218 - in /nutch/branches/2.x: CHANGES.txt src/java/org/apache/nutch/fetcher/FetcherJob.java

[ta...@apache.org]

Author: talat
Date: Sat May  3 15:26:38 2014
New Revision: 1592218

URL: http://svn.apache.org/r1592218
Log:
NUTCH-1618 Turn speculative execution off for Fetching (talat)


Modified:
    nutch/branches/2.x/CHANGES.txt
    nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java

Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1592218&r1=1592217&r2=1592218&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Sat May  3 15:26:38 2014
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Current Development
 
+* NUTCH-1618 Turn speculative execution off for Fetching (talat)
+
 * NUTCH-1657 ORIGINAL_CHAR_ENCODING and CHAR_ENCODING_FOR_CONVERSION never set in HTMLParser (talat)
 
 * NUTCH-1725 CleaningJob's reducer does not commit deleted docs. (ilhamikalkan via talat)

Modified: nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java?rev=1592218&r1=1592217&r2=1592218&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherJob.java Sat May  3 15:26:38 2014
@@ -183,6 +183,10 @@ public class FetcherJob extends NutchToo
     LOG.info("FetcherJob : timelimit set for : " + getConf().getLong("fetcher.timelimit", -1));
     numJobs = 1;
     currentJob = new NutchJob(getConf(), "fetch");
+    
+    // for politeness, don't permit parallel execution of a single task
+    currentJob.setReduceSpeculativeExecution(false);
+    
     Collection<WebPage.Field> fields = getFields(currentJob);
     StorageUtils.initMapperJob(currentJob, fields, IntWritable.class,
         FetchEntry.class, FetcherMapper.class, FetchEntryPartitioner.class, false);