svn commit: r1039578 - /mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java

[ss...@apache.org]

Author: ssc
Date: Fri Nov 26 22:20:48 2010
New Revision: 1039578

URL: http://svn.apache.org/viewvc?rev=1039578&view=rev
Log:
MAHOUT-540 Integration test for the DocumentProcessor

Added:
    mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java

Added: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java?rev=1039578&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java Fri Nov 26 22:20:48 2010
@@ -0,0 +1,54 @@
+package org.apache.mahout.vectorizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.common.StringTuple;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+/**
+ * Tests tokenizing of <Text documentId, Text text> {@link SequenceFile}s by the {@link DocumentProcessor} into
+ * <Text documentId, StringTuple tokens> sequence files
+ */
+public class DocumentProcessorTest extends MahoutTestCase {
+
+  @Test
+  public void testTokenizeDocuments() throws Exception {
+    Configuration configuration = new Configuration();
+    FileSystem fs = FileSystem.get(configuration);
+    Path input = new Path(getTestTempDirPath(), "inputDir");
+    Path output = new Path(getTestTempDirPath(), "outputDir");
+
+    String documentId1 = "123";
+    String text1 = "A test for the document processor";
+    String documentId2 = "456";
+    String text2 = "and another one";
+
+    SequenceFile.Writer writer = new SequenceFile.Writer(fs, configuration, input, Text.class, Text.class);
+    writer.append(new Text(documentId1), new Text(text1));
+    writer.append(new Text(documentId2), new Text(text2));
+    writer.close();
+
+    DocumentProcessor.tokenizeDocuments(input, DefaultAnalyzer.class, output);
+
+    FileStatus[] statuses = fs.listStatus(output);
+    assertEquals(1, statuses.length);
+    Path filePath = statuses[0].getPath();
+    SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, configuration);
+    Text key = reader.getKeyClass().asSubclass(Text.class).newInstance();
+    StringTuple value = reader.getValueClass().asSubclass(StringTuple.class).newInstance();
+
+    reader.next(key, value);
+    assertEquals(documentId1, key.toString());
+    assertEquals(Arrays.asList("test", "document", "processor"), value.getEntries());
+    reader.next(key, value);
+    assertEquals(documentId2, key.toString());
+    assertEquals(Arrays.asList("another", "one"), value.getEntries());
+  }
+}