svn commit: r788024 - in /lucene/mahout/trunk/utils/src: main/java/org/apache/mahout/utils/vectors/ test/java/org/apache/mahout/utils/vectors/

[gs...@apache.org]

Author: gsingers
Date: Wed Jun 24 13:58:20 2009
New Revision: 788024

URL: http://svn.apache.org/viewvc?rev=788024&view=rev
Log:
VectorIterable from a SequenceFile

Added:
    lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java
    lucene/mahout/trunk/utils/src/test/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterableTest.java
Modified:
    lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/Driver.java

Modified: lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/Driver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/Driver.java?rev=788024&r1=788023&r2=788024&view=diff
==============================================================================
--- lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/Driver.java (original)
+++ lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/Driver.java Wed Jun 24 13:58:20 2009
@@ -187,8 +187,9 @@
             vectorWriter = getSeqFileWriter(outFile);
           }
 
-          vectorWriter.write(iteratable, maxDocs);
+          long numDocs = vectorWriter.write(iteratable, maxDocs);
           vectorWriter.close();
+          log.info("Wrote: " + numDocs + " vectors");
 
           String delimiter = cmdLine.hasOption(delimiterOpt) ? cmdLine.getValue(delimiterOpt).toString() : "\t";
           File dictOutFile = new File(cmdLine.getValue(dictOutOpt).toString());

Added: lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java?rev=788024&view=auto
==============================================================================
--- lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java (added)
+++ lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java Wed Jun 24 13:58:20 2009
@@ -0,0 +1,62 @@
+package org.apache.mahout.utils.vectors;
+
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.mahout.matrix.Vector;
+
+import java.util.Iterator;
+import java.io.IOException;
+
+
+/**
+ *
+ *
+ **/
+public class SequenceFileVectorIterable implements VectorIterable {
+  private SequenceFile.Reader reader;
+
+  public SequenceFileVectorIterable(SequenceFile.Reader reader) {
+    this.reader = reader;
+  }
+
+
+  @Override
+  public Iterator<Vector> iterator() {
+    try {
+      return new SeqFileIterator();
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException(e);
+    } catch (InstantiationException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private class SeqFileIterator implements Iterator<Vector> {
+    private Writable key;
+    private Vector value;
+
+    private SeqFileIterator() throws IllegalAccessException, InstantiationException {
+      value = (Vector) reader.getValueClass().newInstance();
+      key = (Writable) reader.getKeyClass().newInstance();
+    }
+
+    @Override
+    public boolean hasNext() {
+      try {
+        return reader.next(key, value);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    @Override
+    public Vector next() {
+      return value;
+    }
+
+    @Override
+    public void remove() {
+      throw new UnsupportedOperationException();
+    }
+  }
+}

Added: lucene/mahout/trunk/utils/src/test/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterableTest.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/test/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterableTest.java?rev=788024&view=auto
==============================================================================
--- lucene/mahout/trunk/utils/src/test/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterableTest.java (added)
+++ lucene/mahout/trunk/utils/src/test/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterableTest.java Wed Jun 24 13:58:20 2009
@@ -0,0 +1,47 @@
+package org.apache.mahout.utils.vectors;
+
+import junit.framework.TestCase;
+
+import java.io.File;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.mahout.matrix.SparseVector;
+import org.apache.mahout.matrix.Vector;
+import org.apache.mahout.utils.vectors.io.SequenceFileVectorWriter;
+
+
+/**
+ *
+ *
+ **/
+public class SequenceFileVectorIterableTest extends TestCase {
+  public void testIterable() throws Exception {
+    File tmpDir = new File(System.getProperty("java.io.tmpdir"));
+    File tmpLoc = new File(tmpDir, "sfvit");
+    tmpLoc.mkdirs();
+    File tmpFile = File.createTempFile("sfvit", ".dat", tmpLoc);
+
+    Path path = new Path(tmpFile.getAbsolutePath());
+    Configuration conf = new Configuration();
+    FileSystem fs = FileSystem.get(conf);
+    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, LongWritable.class, SparseVector.class);
+    SequenceFileVectorWriter writer = new SequenceFileVectorWriter(seqWriter);
+    RandomVectorIterable iter = new RandomVectorIterable(50);
+    writer.write(iter);
+    writer.close();
+
+    SequenceFile.Reader seqReader = new SequenceFile.Reader(fs, path, conf);
+    SequenceFileVectorIterable sfvi = new SequenceFileVectorIterable(seqReader);
+    int count = 0;
+    for (Vector vector : sfvi) {
+      System.out.println("Vec: " + vector.asFormatString());
+      count++;
+    }
+    seqReader.close();
+    assertTrue(count + " does not equal: " + 50, count == 50);
+  }
+}