You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by cu...@apache.org on 2005/08/26 18:37:59 UTC
svn commit: r240279 - in
/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred:
MapFileOutputFormat.java MapTask.java RecordWriter.java ReduceTask.java
SequenceFileOutputFormat.java TaskTracker.java TextOutputFormat.java
Author: cutting
Date: Fri Aug 26 09:37:55 2005
New Revision: 240279
URL: http://svn.apache.org/viewcvs?rev=240279&view=rev
Log:
Always call done() on tasks, setting final progress to 1.0. Also permit RecordWriter.close() to emit progress reports to avoid task timeouts when closing is lengthy.
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapFileOutputFormat.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapTask.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/RecordWriter.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapFileOutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapFileOutputFormat.java?rev=240279&r1=240278&r2=240279&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapFileOutputFormat.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapFileOutputFormat.java Fri Aug 26 09:37:55 2005
@@ -46,7 +46,7 @@
out.append(key, value);
}
- public void close() throws IOException { out.close(); }
+ public void close(Reporter reporter) throws IOException { out.close();}
};
}
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapTask.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapTask.java?rev=240279&r1=240278&r2=240279&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapTask.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/MapTask.java Fri Aug 26 09:37:55 2005
@@ -118,8 +118,6 @@
((CombiningCollector)collector).flush();
}
- reportProgress(umbilical, 1.0f); // done
-
} finally {
in.close(); // close input
}
@@ -130,6 +128,7 @@
}
}
}
+ umbilical.done(getTaskId());
}
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/RecordWriter.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/RecordWriter.java?rev=240279&r1=240278&r2=240279&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/RecordWriter.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/RecordWriter.java Fri Aug 26 09:37:55 2005
@@ -35,5 +35,5 @@
void write(WritableComparable key, Writable value) throws IOException;
/** Close this to future operations.*/
- void close() throws IOException;
+ void close(Reporter reporter) throws IOException;
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java?rev=240279&r1=240278&r2=240279&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java Fri Aug 26 09:37:55 2005
@@ -264,10 +264,10 @@
} finally {
in.close();
lfs.delete(new File(sortedFile)); // remove sorted
- out.close();
+ out.close(reporter);
}
- reportProgress(umbilical);
+ umbilical.done(getTaskId());
}
/** Construct output file names so that, when an output directory listing is
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java?rev=240279&r1=240278&r2=240279&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java Fri Aug 26 09:37:55 2005
@@ -46,7 +46,7 @@
out.append(key, value);
}
- public void close() throws IOException { out.close(); }
+ public void close(Reporter reporter) throws IOException { out.close();}
};
}
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java?rev=240279&r1=240278&r2=240279&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java Fri Aug 26 09:37:55 2005
@@ -408,6 +408,7 @@
*/
public synchronized void reportDone() {
LOG.info("Task " + task.getTaskId() + " is done.");
+ this.progress = 1.0f;
this.done = true;
}
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java?rev=240279&r1=240278&r2=240279&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java Fri Aug 26 09:37:55 2005
@@ -43,7 +43,7 @@
out.writeBytes(value.toString()); // BUG: assume 8-bit chars
out.writeByte('\n');
}
- public synchronized void close() throws IOException {
+ public synchronized void close(Reporter reporter) throws IOException {
out.close();
}
};