You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by fr...@apache.org on 2014/04/28 22:14:20 UTC
svn commit: r1590774 - in /mahout/trunk: ./
integration/src/main/java/org/apache/mahout/text/
integration/src/test/java/org/apache/mahout/text/
integration/src/test/java/org/apache/mahout/text/doc/
Author: frankscholten
Date: Mon Apr 28 20:14:19 2014
New Revision: 1590774
URL: http://svn.apache.org/r1590774
Log:
MAHOUT-1521 lucene2seq - Error trying to load data from stored field (when non-indexed).
Added:
mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java
Modified:
mahout/trunk/CHANGELOG
mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java
mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java
mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java
mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java
Modified: mahout/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/mahout/trunk/CHANGELOG?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/CHANGELOG (original)
+++ mahout/trunk/CHANGELOG Mon Apr 28 20:14:19 2014
@@ -2,6 +2,8 @@ Mahout Change Log
Release 1.0 - unreleased
+ MAHOUT-1521: lucene2seq - Error trying to load data from stored field (when non-indexed) (Terry Blankers via frankscholten)
+
MAHOUT-1310: Mahout support windows (Sergey Svinarchuk via ssc)
MAHOUT-1520: Fix links in Mahout website documentation (Saleem Ansari via smarthi)
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneIndexHelper.java Mon Apr 28 20:14:19 2014
@@ -16,13 +16,13 @@ package org.apache.mahout.text;
* limitations under the License.
*/
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
import java.io.IOException;
/**
- * Utility for checking if a field exist in a Lucene index.
+ * Utility for checking if a field is stored in a Lucene index.
*/
public class LuceneIndexHelper {
@@ -30,10 +30,10 @@ public class LuceneIndexHelper {
}
- public static void fieldShouldExistInIndex(IndexSearcher searcher, String field) throws IOException {
- CollectionStatistics idFieldStatistics = searcher.collectionStatistics(field);
- if (idFieldStatistics.docCount() == 0) {
- throw new IllegalArgumentException("Field '" + field + "' does not exist in the index");
+ public static void fieldShouldExistInIndex(IndexReader reader, String fieldName) throws IOException {
+ IndexableField field = reader.document(0).getField(fieldName);
+ if (field == null || !field.fieldType().stored()) {
+ throw new IllegalArgumentException("Field '" + fieldName + "' is possibly not stored since first document in index does not contain this field.");
}
}
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneSegmentRecordReader.java Mon Apr 28 20:14:19 2014
@@ -58,9 +58,9 @@ public class LuceneSegmentRecordReader e
IndexSearcher searcher = new IndexSearcher(segmentReader);
String idField = lucene2SeqConfiguration.getIdField();
- LuceneIndexHelper.fieldShouldExistInIndex(searcher, idField);
+ LuceneIndexHelper.fieldShouldExistInIndex(segmentReader, idField);
for (String field : lucene2SeqConfiguration.getFields()) {
- LuceneIndexHelper.fieldShouldExistInIndex(searcher, field);
+ LuceneIndexHelper.fieldShouldExistInIndex(segmentReader, field);
}
Weight weight = lucene2SeqConfiguration.getQuery().createWeight(searcher);
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorage.java Mon Apr 28 20:14:19 2014
@@ -64,9 +64,9 @@ public class SequenceFilesFromLuceneStor
IndexReader reader = DirectoryReader.open(directory);
IndexSearcher searcher = new IndexSearcher(reader);
- LuceneIndexHelper.fieldShouldExistInIndex(searcher, lucene2seqConf.getIdField());
+ LuceneIndexHelper.fieldShouldExistInIndex(reader, lucene2seqConf.getIdField());
for (String field : lucene2seqConf.getFields()) {
- LuceneIndexHelper.fieldShouldExistInIndex(searcher, field);
+ LuceneIndexHelper.fieldShouldExistInIndex(reader, field);
}
Configuration configuration = lucene2seqConf.getConfiguration();
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java Mon Apr 28 20:14:19 2014
@@ -31,6 +31,7 @@ import org.apache.mahout.common.Pair;
import org.apache.mahout.text.doc.MultipleFieldsDocument;
import org.apache.mahout.text.doc.NumericFieldDocument;
import org.apache.mahout.text.doc.SingleFieldDocument;
+import org.apache.mahout.text.doc.TestDocument;
import java.io.File;
import java.io.IOException;
@@ -44,8 +45,8 @@ public abstract class AbstractLuceneStor
protected Path indexPath1;
protected Path indexPath2;
- protected List<SingleFieldDocument> docs = Lists.newArrayList();
- protected List<SingleFieldDocument> misshapenDocs = Lists.newArrayList();
+ protected List<TestDocument> docs = Lists.newArrayList();
+ protected List<TestDocument> misshapenDocs = Lists.newArrayList();
@Override
public void setUp() throws Exception {
@@ -59,10 +60,10 @@ public abstract class AbstractLuceneStor
misshapenDocs.add(new SingleFieldDocument("empty_value", ""));
}
- protected void commitDocuments(Directory directory, Iterable<SingleFieldDocument> theDocs) throws IOException{
+ protected void commitDocuments(Directory directory, Iterable<TestDocument> theDocs) throws IOException{
IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_46, new StandardAnalyzer(Version.LUCENE_46)));
- for (SingleFieldDocument singleFieldDocument : theDocs) {
+ for (TestDocument singleFieldDocument : theDocs) {
indexWriter.addDocument(singleFieldDocument.asLuceneDocument());
}
@@ -70,7 +71,7 @@ public abstract class AbstractLuceneStor
indexWriter.close();
}
- protected void commitDocuments(Directory directory, SingleFieldDocument... documents) throws IOException {
+ protected void commitDocuments(Directory directory, TestDocument... documents) throws IOException {
commitDocuments(directory, Arrays.asList(documents));
}
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriverTest.java Mon Apr 28 20:14:19 2014
@@ -65,8 +65,6 @@ public class SequenceFilesFromLuceneStor
return lucene2SeqConf;
}
};
- commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool"));
- commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool"));
}
@After
@@ -127,6 +125,9 @@ public class SequenceFilesFromLuceneStor
@Test
public void testRunOptionalArguments() throws Exception {
+ commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool"));
+ commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool"));
+
String[] args = {
"-i", getIndexPath1AsFile().toString(),
"-o", seqFilesOutputPath.toString(),
@@ -150,6 +151,9 @@ public class SequenceFilesFromLuceneStor
@Test
public void testRunInvalidQuery() throws Exception {
+ commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("1", "Mahout is cool"));
+ commitDocuments(getDirectory(getIndexPath1AsFile()), new SingleFieldDocument("2", "Mahout is cool"));
+
String[] args = {
"-i", getIndexPath1AsFile().toString(),
"-o", seqFilesOutputPath.toString(),
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageMRJobTest.java Mon Apr 28 20:14:19 2014
@@ -23,6 +23,7 @@ import org.apache.hadoop.io.Text;
import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.Pair;
import org.apache.mahout.text.doc.SingleFieldDocument;
+import org.apache.mahout.text.doc.TestDocument;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -71,12 +72,12 @@ public class SequenceFilesFromLuceneStor
map.put(next.getFirst().toString(), next.getSecond());
}
assertEquals(docs.size() + misshapenDocs.size(), map.size());
- for (SingleFieldDocument doc : docs) {
+ for (TestDocument doc : docs) {
Text value = map.get(doc.getId());
assertNotNull(value);
assertEquals(value.toString(), doc.getField());
}
- for (SingleFieldDocument doc : misshapenDocs) {
+ for (TestDocument doc : misshapenDocs) {
Text value = map.get(doc.getId());
assertNotNull(value);
assertEquals(value.toString(), doc.getField());
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageTest.java Mon Apr 28 20:14:19 2014
@@ -20,6 +20,9 @@ import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
@@ -28,6 +31,7 @@ import org.apache.mahout.common.Pair;
import org.apache.mahout.text.doc.MultipleFieldsDocument;
import org.apache.mahout.text.doc.NumericFieldDocument;
import org.apache.mahout.text.doc.SingleFieldDocument;
+import org.apache.mahout.text.doc.TestDocument;
import org.apache.mahout.text.doc.UnstoredFieldsDocument;
import org.junit.After;
import org.junit.Before;
@@ -85,20 +89,20 @@ public class SequenceFilesFromLuceneStor
map.put(next.getFirst().toString(), next.getSecond());
}
assertEquals(docs.size() + misshapenDocs.size(), map.size());
- for (SingleFieldDocument doc : docs) {
+ for (TestDocument doc : docs) {
Text value = map.get(doc.getId());
assertNotNull(value);
assertEquals(value.toString(), doc.getField());
}
- for (SingleFieldDocument doc : misshapenDocs) {
+ for (TestDocument doc : misshapenDocs) {
Text value = map.get(doc.getId());
assertNotNull(value);
assertEquals(value.toString(), doc.getField());
}
}
- @Test
- public void testRunSkipUnstoredFields() throws IOException {
+ @Test(expected = IllegalArgumentException.class)
+ public void testRun_UnstoredFields() throws IOException {
commitDocuments(getDirectory(getIndexPath1AsFile()), new UnstoredFieldsDocument("5", "This is test document 5"));
LuceneStorageConfiguration lucene2SeqConf = new LuceneStorageConfiguration(configuration,
@@ -108,11 +112,6 @@ public class SequenceFilesFromLuceneStor
asList(UnstoredFieldsDocument.FIELD, UnstoredFieldsDocument.UNSTORED_FIELD));
lucene2Seq.run(lucene2SeqConf);
-
- Iterator<Pair<Text, Text>> iterator = lucene2SeqConf.getSequenceFileIterator();
-
- assertFalse(iterator.next().getSecond().toString().contains("null"));
- assertFalse(iterator.hasNext());
}
@Test
@@ -227,4 +226,25 @@ public class SequenceFilesFromLuceneStor
lucene2Seq.run(lucene2SeqConf);
}
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testIndexedButNotStoredField() throws IOException {
+ SingleFieldDocument document = new SingleFieldDocument("id", "field") {
+ @Override
+ public Document asLuceneDocument() {
+ Document document = super.asLuceneDocument();
+ document.add(new TextField("indexed", "This text is indexed", Field.Store.NO));
+ return document;
+ }
+ };
+ commitDocuments(getDirectory(getIndexPath1AsFile()), document);
+
+ lucene2SeqConf = new LuceneStorageConfiguration(configuration,
+ asList(getIndexPath1()),
+ seqFilesOutputPath,
+ SingleFieldDocument.ID_FIELD,
+ asList(SingleFieldDocument.FIELD, "indexed"));
+
+ lucene2Seq.run(lucene2SeqConf);
+ }
}
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java?rev=1590774&r1=1590773&r2=1590774&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/SingleFieldDocument.java Mon Apr 28 20:14:19 2014
@@ -24,7 +24,7 @@ import org.apache.lucene.document.TextFi
/**
* Used for testing lucene2seq
*/
-public class SingleFieldDocument {
+public class SingleFieldDocument implements TestDocument {
public static final String ID_FIELD = "idField";
public static final String FIELD = "field";
@@ -37,14 +37,17 @@ public class SingleFieldDocument {
this.field = field;
}
+ @Override
public String getId() {
return id;
}
+ @Override
public String getField() {
return field;
}
+ @Override
public Document asLuceneDocument() {
Document document = new Document();
Added: mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java?rev=1590774&view=auto
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java (added)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/doc/TestDocument.java Mon Apr 28 20:14:19 2014
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.text.doc;
+
+import org.apache.lucene.document.Document;
+
+public interface TestDocument {
+
+ String getId();
+
+ String getField();
+
+ Document asLuceneDocument();
+
+}