You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by fr...@apache.org on 2014/04/28 22:14:20 UTC

svn commit: r1590774 - in /mahout/trunk: ./ integration/src/main/java/org/apache/mahout/text/ integration/src/test/java/org/apache/mahout/text/ integration/src/test/java/org/apache/mahout/text/doc/

Author: frankscholten
Date: Mon Apr 28 20:14:19 2014
New Revision: 1590774

URL: http://svn.apache.org/r1590774
Log:
MAHOUT-1521 lucene2seq - Error trying to load data from stored field (when non-indexed).

Added:
    mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java
Modified:
    mahout/trunk/CHANGELOG
    mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java
    mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java
    mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java
    mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java
    mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java
    mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java
    mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java
    mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java

Modified: mahout/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/mahout/trunk/CHANGELOG?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/CHANGELOG (original)
+++ mahout/trunk/CHANGELOG Mon Apr 28 20:14:19 2014
@@ -2,6 +2,8 @@ Mahout Change Log
 
 Release 1.0 - unreleased
 
+  MAHOUT-1521: lucene2seq - Error trying to load data from stored field (when non-indexed) (Terry Blankers via frankscholten)
+
   MAHOUT-1310: Mahout support windows (Sergey Svinarchuk via ssc)
 
   MAHOUT-1520: Fix links in Mahout website documentation (Saleem Ansari via smarthi)

Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java Mon Apr 28 20:14:19 2014
@@ -16,13 +16,13 @@ package org.apache.mahout.text;
  * limitations under the License.
  */
 
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 
 import java.io.IOException;
 
 /**
- * Utility for checking if a field exist in a Lucene index.
+ * Utility for checking if a field is stored in a Lucene index.
  */
 public class LuceneIndexHelper {
 
@@ -30,10 +30,10 @@ public class LuceneIndexHelper {
 
   }
 
-  public static void fieldShouldExistInIndex(IndexSearcher searcher, String field) throws IOException {
-    CollectionStatistics idFieldStatistics = searcher.collectionStatistics(field);
-    if (idFieldStatistics.docCount() == 0) {
-      throw new IllegalArgumentException("Field '" + field + "' does not exist in the index");
+  public static void fieldShouldExistInIndex(IndexReader reader, String fieldName) throws IOException {
+    IndexableField field = reader.document(0).getField(fieldName);
+    if (field == null || !field.fieldType().stored()) {
+      throw new IllegalArgumentException("Field '" + fieldName + "' is possibly not stored since first document in index does not contain this field.");
     }
   }
 

Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java Mon Apr 28 20:14:19 2014
@@ -58,9 +58,9 @@ public class LuceneSegmentRecordReader e
 
     IndexSearcher searcher = new IndexSearcher(segmentReader);
     String idField = lucene2SeqConfiguration.getIdField();
-    LuceneIndexHelper.fieldShouldExistInIndex(searcher, idField);
+    LuceneIndexHelper.fieldShouldExistInIndex(segmentReader, idField);
     for (String field : lucene2SeqConfiguration.getFields()) {
-        LuceneIndexHelper.fieldShouldExistInIndex(searcher, field);
+        LuceneIndexHelper.fieldShouldExistInIndex(segmentReader, field);
     }
 
     Weight weight = lucene2SeqConfiguration.getQuery().createWeight(searcher);

Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java Mon Apr 28 20:14:19 2014
@@ -64,9 +64,9 @@ public class SequenceFilesFromLuceneStor
       IndexReader reader = DirectoryReader.open(directory);
       IndexSearcher searcher = new IndexSearcher(reader);
 
-      LuceneIndexHelper.fieldShouldExistInIndex(searcher, lucene2seqConf.getIdField());
+      LuceneIndexHelper.fieldShouldExistInIndex(reader, lucene2seqConf.getIdField());
       for (String field : lucene2seqConf.getFields()) {
-        LuceneIndexHelper.fieldShouldExistInIndex(searcher, field);
+        LuceneIndexHelper.fieldShouldExistInIndex(reader, field);
       }
 
       Configuration configuration = lucene2seqConf.getConfiguration();

Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java Mon Apr 28 20:14:19 2014
@@ -31,6 +31,7 @@ import org.apache.mahout.common.Pair;
 import org.apache.mahout.text.doc.MultipleFieldsDocument;
 import org.apache.mahout.text.doc.NumericFieldDocument;
 import org.apache.mahout.text.doc.SingleFieldDocument;
+import org.apache.mahout.text.doc.TestDocument;
 
 import java.io.File;
 import java.io.IOException;
@@ -44,8 +45,8 @@ public abstract class AbstractLuceneStor
 
   protected Path indexPath1;
   protected Path indexPath2;
-  protected List<SingleFieldDocument> docs = Lists.newArrayList();
-  protected List<SingleFieldDocument> misshapenDocs = Lists.newArrayList();
+  protected List<TestDocument> docs = Lists.newArrayList();
+  protected List<TestDocument> misshapenDocs = Lists.newArrayList();
 
   @Override
   public void setUp() throws Exception {
@@ -59,10 +60,10 @@ public abstract class AbstractLuceneStor
     misshapenDocs.add(new SingleFieldDocument("empty_value", ""));
   }
 
-  protected void commitDocuments(Directory directory, Iterable<SingleFieldDocument> theDocs) throws IOException{
+  protected void commitDocuments(Directory directory, Iterable<TestDocument> theDocs) throws IOException{
     IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_46, new StandardAnalyzer(Version.LUCENE_46)));
 
-    for (SingleFieldDocument singleFieldDocument : theDocs) {
+    for (TestDocument singleFieldDocument : theDocs) {
       indexWriter.addDocument(singleFieldDocument.asLuceneDocument());
     }
 
@@ -70,7 +71,7 @@ public abstract class AbstractLuceneStor
     indexWriter.close();
   }
 
-  protected void commitDocuments(Directory directory, SingleFieldDocument... documents) throws IOException {
+  protected void commitDocuments(Directory directory, TestDocument... documents) throws IOException {
     commitDocuments(directory, Arrays.asList(documents));
   }
 

Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java Mon Apr 28 20:14:19 2014
@@ -65,8 +65,6 @@ public class SequenceFilesFromLuceneStor
         return lucene2SeqConf;
       }
     };
-    commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool"));
-    commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool"));
   }
 
   @After
@@ -127,6 +125,9 @@ public class SequenceFilesFromLuceneStor
 
   @Test
   public void testRunOptionalArguments() throws Exception {
+    commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool"));
+    commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool"));
+
     String[] args = {
       "-i", getIndexPath1AsFile().toString(),
       "-o", seqFilesOutputPath.toString(),
@@ -150,6 +151,9 @@ public class SequenceFilesFromLuceneStor
 
   @Test
   public void testRunInvalidQuery() throws Exception {
+    commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool"));
+    commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool"));
+
     String[] args = {
       "-i", getIndexPath1AsFile().toString(),
       "-o", seqFilesOutputPath.toString(),

Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java Mon Apr 28 20:14:19 2014
@@ -23,6 +23,7 @@ import org.apache.hadoop.io.Text;
 import org.apache.mahout.common.HadoopUtil;
 import org.apache.mahout.common.Pair;
 import org.apache.mahout.text.doc.SingleFieldDocument;
+import org.apache.mahout.text.doc.TestDocument;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -71,12 +72,12 @@ public class SequenceFilesFromLuceneStor
       map.put(next.getFirst().toString(), next.getSecond());
     }
     assertEquals(docs.size() + misshapenDocs.size(), map.size());
-    for (SingleFieldDocument doc : docs) {
+    for (TestDocument doc : docs) {
       Text value = map.get(doc.getId());
       assertNotNull(value);
       assertEquals(value.toString(), doc.getField());
     }
-    for (SingleFieldDocument doc : misshapenDocs) {
+    for (TestDocument doc : misshapenDocs) {
       Text value = map.get(doc.getId());
       assertNotNull(value);
       assertEquals(value.toString(), doc.getField());

Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java Mon Apr 28 20:14:19 2014
@@ -20,6 +20,9 @@ import com.google.common.collect.Maps;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -28,6 +31,7 @@ import org.apache.mahout.common.Pair;
 import org.apache.mahout.text.doc.MultipleFieldsDocument;
 import org.apache.mahout.text.doc.NumericFieldDocument;
 import org.apache.mahout.text.doc.SingleFieldDocument;
+import org.apache.mahout.text.doc.TestDocument;
 import org.apache.mahout.text.doc.UnstoredFieldsDocument;
 import org.junit.After;
 import org.junit.Before;
@@ -85,20 +89,20 @@ public class SequenceFilesFromLuceneStor
       map.put(next.getFirst().toString(), next.getSecond());
     }
     assertEquals(docs.size() + misshapenDocs.size(), map.size());
-    for (SingleFieldDocument doc : docs) {
+    for (TestDocument doc : docs) {
       Text value = map.get(doc.getId());
       assertNotNull(value);
       assertEquals(value.toString(), doc.getField());
     }
-    for (SingleFieldDocument doc : misshapenDocs) {
+    for (TestDocument doc : misshapenDocs) {
       Text value = map.get(doc.getId());
       assertNotNull(value);
       assertEquals(value.toString(), doc.getField());
     }
   }
 
-  @Test
-  public void testRunSkipUnstoredFields() throws IOException {
+  @Test(expected = IllegalArgumentException.class)
+  public void testRun_UnstoredFields() throws IOException {
     commitDocuments(getDirectory(getIndexPath1AsFile()), new UnstoredFieldsDocument("5", "This is test document 5"));
 
     LuceneStorageConfiguration lucene2SeqConf = new LuceneStorageConfiguration(configuration,
@@ -108,11 +112,6 @@ public class SequenceFilesFromLuceneStor
       asList(UnstoredFieldsDocument.FIELD, UnstoredFieldsDocument.UNSTORED_FIELD));
 
     lucene2Seq.run(lucene2SeqConf);
-
-    Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator();
-
-    assertFalse(iterator.next().getSecond().toString().contains("null"));
-    assertFalse(iterator.hasNext());
   }
 
   @Test
@@ -227,4 +226,25 @@ public class SequenceFilesFromLuceneStor
 
     lucene2Seq.run(lucene2SeqConf);
   }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testIndexedButNotStoredField() throws IOException {
+    SingleFieldDocument document = new SingleFieldDocument("id", "field") {
+      @Override
+      public Document asLuceneDocument() {
+        Document document = super.asLuceneDocument();
+        document.add(new TextField("indexed", "This text is indexed", Field.Store.NO));
+        return document;
+      }
+    };
+    commitDocuments(getDirectory(getIndexPath1AsFile()), document);
+
+    lucene2SeqConf = new LuceneStorageConfiguration(configuration,
+        asList(getIndexPath1()),
+        seqFilesOutputPath,
+        SingleFieldDocument.ID_FIELD,
+        asList(SingleFieldDocument.FIELD, "indexed"));
+
+    lucene2Seq.run(lucene2SeqConf);
+  }
 }

Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java Mon Apr 28 20:14:19 2014
@@ -24,7 +24,7 @@ import org.apache.lucene.document.TextFi
 /**
  * Used for testing lucene2seq
  */
-public class SingleFieldDocument {
+public class SingleFieldDocument implements TestDocument {
 
   public static final String ID_FIELD = "idField";
   public static final String FIELD = "field";
@@ -37,14 +37,17 @@ public class SingleFieldDocument {
     this.field = field;
   }
 
+  @Override
   public String getId() {
     return id;
   }
 
+  @Override
   public String getField() {
     return field;
   }
 
+  @Override
   public Document asLuceneDocument() {
     Document document = new Document();
 

Added: mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java?rev=1590774&view=auto
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java (added)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java Mon Apr 28 20:14:19 2014
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.text.doc;
+
+import org.apache.lucene.document.Document;
+
+public interface TestDocument {
+
+  String getId();
+
+  String getField();
+
+  Document asLuceneDocument();
+
+}