You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by cu...@apache.org on 2005/09/02 01:24:17 UTC
svn commit: r265808 -
/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java
Author: cutting
Date: Thu Sep 1 16:24:15 2005
New Revision: 265808
URL: http://svn.apache.org/viewcvs?rev=265808&view=rev
Log:
Use comparator when comparing values in reduce.
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java
Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java?rev=265808&r1=265807&r2=265808&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/ReduceTask.java Thu Sep 1 16:24:15 2005
@@ -87,13 +87,16 @@
private boolean more; // more in file
private float progPerByte;
private TaskUmbilicalProtocol umbilical;
+ private WritableComparator comparator;
public ValuesIterator (SequenceFile.Reader in, long length,
+ WritableComparator comparator,
TaskUmbilicalProtocol umbilical)
throws IOException {
this.in = in;
this.progPerByte = 1.0f / (float)length;
this.umbilical = umbilical;
+ this.comparator = comparator;
getNext();
}
@@ -143,7 +146,7 @@
if (lastKey == null) {
hasNext = true;
} else {
- hasNext = (key.compareTo(lastKey) == 0);
+ hasNext = (comparator.compare(key, lastKey) == 0);
}
} else {
hasNext = false;
@@ -220,13 +223,13 @@
String sortedFile = file+".sorted";
+ WritableComparator comparator =
+ (WritableComparator)job.newInstance(job.getOutputKeyComparatorClass());
+
try {
sortProgress.start();
// sort the input file
- WritableComparator comparator =
- (WritableComparator)job.newInstance(job.getOutputKeyComparatorClass());
-
SequenceFile.Sorter sorter =
new SequenceFile.Sorter(lfs, comparator, valueClass);
sorter.sort(file, sortedFile); // sort
@@ -255,7 +258,8 @@
Reporter reporter = getReporter(umbilical, getProgress());
long length = lfs.getLength(new File(sortedFile));
try {
- ValuesIterator values = new ValuesIterator(in, length, umbilical);
+ ValuesIterator values = new ValuesIterator(in, length, comparator,
+ umbilical);
while (values.more()) {
reducer.reduce(values.getKey(), values, collector, reporter);
values.nextKey();