svn commit: r348427 - in /lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred: SequenceFileOutputFormat.java TextOutputFormat.java

[ab...@apache.org]

Author: ab
Date: Wed Nov 23 05:12:56 2005
New Revision: 348427

URL: http://svn.apache.org/viewcvs?rev=348427&view=rev
Log:
Add a convenience method to get Reader[] (similar as in MapFileOutputFormat).

Make TextOutputFormat output in UTF-8.

Modified:
    lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java
    lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java

Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java?rev=348427&r1=348426&r2=348427&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/SequenceFileOutputFormat.java Wed Nov 23 05:12:56 2005
@@ -18,9 +18,11 @@
 
 import java.io.IOException;
 import java.io.File;
+import java.util.Arrays;
 
 import org.apache.nutch.fs.NutchFileSystem;
 
+import org.apache.nutch.io.MapFile;
 import org.apache.nutch.io.SequenceFile;
 import org.apache.nutch.io.WritableComparable;
 import org.apache.nutch.io.Writable;
@@ -48,6 +50,21 @@
 
         public void close(Reporter reporter) throws IOException { out.close();}
       };
-  }      
+  }
+
+  /** Open the output generated by this format. */
+  public static SequenceFile.Reader[] getReaders(NutchFileSystem fs, File dir)
+    throws IOException {
+    File[] names = fs.listFiles(dir);
+    
+    // sort names, so that hash partitioning works
+    Arrays.sort(names);
+    
+    SequenceFile.Reader[] parts = new SequenceFile.Reader[names.length];
+    for (int i = 0; i < names.length; i++) {
+      parts[i] = new SequenceFile.Reader(fs, names[i].toString());
+    }
+    return parts;
+  }
 }
 

Modified: lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java
URL: http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java?rev=348427&r1=348426&r2=348427&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java (original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/mapred/TextOutputFormat.java Wed Nov 23 05:12:56 2005
@@ -37,9 +37,9 @@
     return new RecordWriter() {
         public synchronized void write(WritableComparable key, Writable value)
           throws IOException {
-          out.writeBytes(key.toString());         // BUG: assume 8-bit chars
+          out.write(key.toString().getBytes("UTF-8"));
           out.writeByte('\t');
-          out.writeBytes(value.toString());       // BUG: assume 8-bit chars
+          out.write(value.toString().getBytes("UTF-8"));
           out.writeByte('\n');
         }
         public synchronized void close(Reporter reporter) throws IOException {