You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/16 20:09:42 UTC
svn commit: r1202842 [4/6] - in /lucene/dev/trunk: ./ dev-tools/eclipse/
dev-tools/idea/lucene/contrib/ dev-tools/idea/lucene/contrib/instantiated/
dev-tools/maven/lucene/contrib/
dev-tools/maven/lucene/contrib/instantiated/ lucene/ lucene/contrib/ luc...
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java Wed Nov 16 19:09:35 2011
@@ -249,7 +249,7 @@ public class TestCodecs extends LuceneTe
terms[i] = new TermData(text, docs, null);
}
- final FieldInfos fieldInfos = new FieldInfos();
+ final FieldInfos fieldInfos = new FieldInfos(new FieldInfos.FieldNumberBiMap());
final FieldData field = new FieldData("field", fieldInfos, terms, true, false);
final FieldData[] fields = new FieldData[] {field};
@@ -264,7 +264,10 @@ public class TestCodecs extends LuceneTe
final FieldsEnum fieldsEnum = reader.iterator();
assertNotNull(fieldsEnum.next());
- final TermsEnum termsEnum = fieldsEnum.terms();
+ final Terms terms2 = fieldsEnum.terms();
+ assertNotNull(terms2);
+
+ final TermsEnum termsEnum = terms2.iterator(null);
DocsEnum docsEnum = null;
for(int i=0;i<NUM_TERMS;i++) {
@@ -293,7 +296,7 @@ public class TestCodecs extends LuceneTe
}
public void testRandomPostings() throws Throwable {
- final FieldInfos fieldInfos = new FieldInfos();
+ final FieldInfos fieldInfos = new FieldInfos(new FieldInfos.FieldNumberBiMap());
final FieldData[] fields = new FieldData[NUM_FIELDS];
for(int i=0;i<NUM_FIELDS;i++) {
@@ -452,7 +455,7 @@ public class TestCodecs extends LuceneTe
for(int iter=0;iter<NUM_TEST_ITER;iter++) {
final FieldData field = fields[TestCodecs.random.nextInt(fields.length)];
- final TermsEnum termsEnum = termsDict.terms(field.fieldInfo.name).iterator();
+ final TermsEnum termsEnum = termsDict.terms(field.fieldInfo.name).iterator(null);
if (si.getCodec() instanceof Lucene3xCodec) {
// code below expects unicode sort order
continue;
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java Wed Nov 16 19:09:35 2011
@@ -85,8 +85,8 @@ public class TestDirectoryReader extends
Document newDoc2 = reader.document(1);
assertTrue(newDoc2 != null);
assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
- TermFreqVector vector = reader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);
- assertTrue(vector != null);
+ Terms vector = reader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
+ assertNotNull(vector);
TestSegmentReader.checkNorms(reader);
reader.close();
}
@@ -165,14 +165,14 @@ public class TestDirectoryReader extends
MultiReader mr3 = new MultiReader(readers2);
// test mixing up TermDocs and TermEnums from different readers.
- TermsEnum te2 = MultiFields.getTerms(mr2, "body").iterator();
+ TermsEnum te2 = MultiFields.getTerms(mr2, "body").iterator(null);
te2.seekCeil(new BytesRef("wow"));
DocsEnum td = MultiFields.getTermDocsEnum(mr2,
MultiFields.getLiveDocs(mr2),
"body",
te2.term());
- TermsEnum te3 = MultiFields.getTerms(mr3, "body").iterator();
+ TermsEnum te3 = MultiFields.getTerms(mr3, "body").iterator(null);
te3.seekCeil(new BytesRef("wow"));
td = te3.docs(MultiFields.getLiveDocs(mr3),
td);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDoc.java Wed Nov 16 19:09:35 2011
@@ -130,13 +130,13 @@ public class TestDoc extends LuceneTestC
printSegment(out, si2);
writer.close();
- SegmentInfo siMerge = merge(si1, si2, "merge", false);
+ SegmentInfo siMerge = merge(directory, si1, si2, "merge", false);
printSegment(out, siMerge);
- SegmentInfo siMerge2 = merge(si1, si2, "merge2", false);
+ SegmentInfo siMerge2 = merge(directory, si1, si2, "merge2", false);
printSegment(out, siMerge2);
- SegmentInfo siMerge3 = merge(siMerge, siMerge2, "merge3", false);
+ SegmentInfo siMerge3 = merge(directory, siMerge, siMerge2, "merge3", false);
printSegment(out, siMerge3);
directory.close();
@@ -164,13 +164,13 @@ public class TestDoc extends LuceneTestC
printSegment(out, si2);
writer.close();
- siMerge = merge(si1, si2, "merge", true);
+ siMerge = merge(directory, si1, si2, "merge", true);
printSegment(out, siMerge);
- siMerge2 = merge(si1, si2, "merge2", true);
+ siMerge2 = merge(directory, si1, si2, "merge2", true);
printSegment(out, siMerge2);
- siMerge3 = merge(siMerge, siMerge2, "merge3", true);
+ siMerge3 = merge(directory, siMerge, siMerge2, "merge3", true);
printSegment(out, siMerge3);
directory.close();
@@ -193,14 +193,14 @@ public class TestDoc extends LuceneTestC
}
- private SegmentInfo merge(SegmentInfo si1, SegmentInfo si2, String merged, boolean useCompoundFile)
+ private SegmentInfo merge(Directory dir, SegmentInfo si1, SegmentInfo si2, String merged, boolean useCompoundFile)
throws Exception {
IOContext context = newIOContext(random);
SegmentReader r1 = SegmentReader.get(true, si1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, context);
SegmentReader r2 = SegmentReader.get(true, si2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, context);
final Codec codec = Codec.getDefault();
- SegmentMerger merger = new SegmentMerger(InfoStream.getDefault(), si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, MergeState.CheckAbort.NONE, null, new FieldInfos(), codec, context);
+ SegmentMerger merger = new SegmentMerger(InfoStream.getDefault(), si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, MergeState.CheckAbort.NONE, null, new FieldInfos(new FieldInfos.FieldNumberBiMap()), codec, context);
merger.add(r1);
merger.add(r2);
@@ -212,7 +212,7 @@ public class TestDoc extends LuceneTestC
false, codec, fieldInfos);
if (useCompoundFile) {
- Collection<String> filesToDelete = merger.createCompoundFile(merged + ".cfs", info, newIOContext(random));
+ Collection<String> filesToDelete = IndexWriter.createCompoundFile(dir, merged + ".cfs", MergeState.CheckAbort.NONE, info, newIOContext(random));
info.setUseCompoundFile(true);
for (final String fileToDelete : filesToDelete)
si1.dir.deleteFile(fileToDelete);
@@ -232,7 +232,9 @@ public class TestDoc extends LuceneTestC
FieldsEnum fis = reader.fields().iterator();
String field = fis.next();
while(field != null) {
- TermsEnum tis = fis.terms();
+ Terms terms = fis.terms();
+ assertNotNull(terms);
+ TermsEnum tis = terms.iterator(null);
while(tis.next() != null) {
out.print(" term=" + field + ":" + tis.term());
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocCount.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocCount.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocCount.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocCount.java Wed Nov 16 19:09:35 2011
@@ -70,7 +70,7 @@ public class TestDocCount extends Lucene
Terms terms = fields.terms(field);
int docCount = terms.getDocCount();
FixedBitSet visited = new FixedBitSet(ir.maxDoc());
- TermsEnum te = terms.iterator();
+ TermsEnum te = terms.iterator(null);
while (te.next() != null) {
DocsEnum de = te.docs(null, null);
while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java Wed Nov 16 19:09:35 2011
@@ -314,7 +314,7 @@ public class TestDocTermOrds extends Luc
if (VERBOSE) {
System.out.println("TEST: verify prefix=" + (prefixRef==null ? "null" : prefixRef.utf8ToString()));
System.out.println("TEST: all TERMS:");
- TermsEnum allTE = MultiFields.getTerms(r, "field").iterator();
+ TermsEnum allTE = MultiFields.getTerms(r, "field").iterator(null);
int ord = 0;
while(allTE.next() != null) {
System.out.println(" ord=" + (ord++) + " term=" + allTE.term().utf8ToString());
@@ -329,7 +329,7 @@ public class TestDocTermOrds extends Luc
} else {
Terms terms = MultiFields.getTerms(r, "field");
if (terms != null) {
- TermsEnum termsEnum = terms.iterator();
+ TermsEnum termsEnum = terms.iterator(null);
TermsEnum.SeekStatus result = termsEnum.seekCeil(prefixRef, false);
if (result != TermsEnum.SeekStatus.END) {
assertFalse("term=" + termsEnum.term().utf8ToString() + " matches prefix=" + prefixRef.utf8ToString(), termsEnum.term().startsWith(prefixRef));
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Wed Nov 16 19:09:35 2011
@@ -287,13 +287,13 @@ public class TestDocumentWriter extends
IndexReader reader = IndexReader.open(dir, true);
// f1
- TermFreqVector tfv1 = reader.getTermFreqVector(0, "f1");
+ Terms tfv1 = reader.getTermVectors(0).terms("f1");
assertNotNull(tfv1);
- assertEquals("the 'with_tv' setting should rule!",2,tfv1.getTerms().length);
+ assertEquals("the 'with_tv' setting should rule!",2,tfv1.getUniqueTermCount());
// f2
- TermFreqVector tfv2 = reader.getTermFreqVector(0, "f2");
+ Terms tfv2 = reader.getTermVectors(0).terms("f2");
assertNotNull(tfv2);
- assertEquals("the 'with_tv' setting should rule!",2,tfv2.getTerms().length);
+ assertEquals("the 'with_tv' setting should rule!",2,tfv2.getUniqueTermCount());
reader.close();
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Wed Nov 16 19:09:35 2011
@@ -21,8 +21,13 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.FieldInfosReader;
+import org.apache.lucene.index.codecs.FieldInfosWriter;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
+import org.junit.Ignore;
import java.io.IOException;
import java.util.Arrays;
@@ -42,7 +47,7 @@ public class TestFieldInfos extends Luce
public FieldInfos createAndWriteFieldInfos(Directory dir, String filename) throws IOException{
//Positive test of FieldInfos
assertTrue(testDoc != null);
- FieldInfos fieldInfos = new FieldInfos();
+ FieldInfos fieldInfos = new FieldInfos(new FieldInfos.FieldNumberBiMap());
_TestUtil.add(testDoc, fieldInfos);
//Since the complement is stored as well in the fields map
assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor
@@ -52,16 +57,23 @@ public class TestFieldInfos extends Luce
assertTrue(output != null);
//Use a RAMOutputStream
- fieldInfos.write(output);
+ FieldInfosWriter writer = Codec.getDefault().fieldInfosFormat().getFieldInfosWriter();
+ writer.write(dir, filename, fieldInfos, IOContext.DEFAULT);
output.close();
return fieldInfos;
}
+
+ public FieldInfos readFieldInfos(Directory dir, String filename) throws IOException {
+ FieldInfosReader reader = Codec.getDefault().fieldInfosFormat().getFieldInfosReader();
+ return reader.read(dir, filename, IOContext.DEFAULT);
+ }
+
public void test() throws IOException {
String name = "testFile";
Directory dir = newDirectory();
FieldInfos fieldInfos = createAndWriteFieldInfos(dir, name);
- assertTrue(dir.fileLength(name) > 0);
- FieldInfos readIn = new FieldInfos(dir, name);
+
+ FieldInfos readIn = readFieldInfos(dir, name);
assertTrue(fieldInfos.size() == readIn.size());
FieldInfo info = readIn.fieldInfo("textField1");
assertTrue(info != null);
@@ -90,7 +102,7 @@ public class TestFieldInfos extends Luce
String name = "testFile";
Directory dir = newDirectory();
FieldInfos fieldInfos = createAndWriteFieldInfos(dir, name);
- FieldInfos readOnly = new FieldInfos(dir, name);
+ FieldInfos readOnly = readFieldInfos(dir, name);
assertReadOnly(readOnly, fieldInfos);
FieldInfos readOnlyClone = (FieldInfos)readOnly.clone();
assertNotSame(readOnly, readOnlyClone);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Wed Nov 16 19:09:35 2011
@@ -48,7 +48,7 @@ public class TestFieldsReader extends Lu
@BeforeClass
public static void beforeClass() throws Exception {
- fieldInfos = new FieldInfos();
+ fieldInfos = new FieldInfos(new FieldInfos.FieldNumberBiMap());
DocHelper.setupDoc(testDoc);
_TestUtil.add(testDoc, fieldInfos);
dir = newDirectory();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java Wed Nov 16 19:09:35 2011
@@ -54,8 +54,8 @@ public class TestFilterIndexReader exten
}
@Override
- public TermsEnum iterator() throws IOException {
- return new TestTermsEnum(super.iterator());
+ public TermsEnum iterator(TermsEnum reuse) throws IOException {
+ return new TestTermsEnum(super.iterator(reuse));
}
}
@@ -65,8 +65,8 @@ public class TestFilterIndexReader exten
}
@Override
- public TermsEnum terms() throws IOException {
- return new TestTermsEnum(super.terms());
+ public Terms terms() throws IOException {
+ return new TestTerms(super.terms());
}
}
@@ -150,7 +150,7 @@ public class TestFilterIndexReader exten
reader.close();
reader = IndexReader.open(target, true);
- TermsEnum terms = MultiFields.getTerms(reader, "default").iterator();
+ TermsEnum terms = MultiFields.getTerms(reader, "default").iterator(null);
while (terms.next() != null) {
assertTrue(terms.term().utf8ToString().indexOf('e') != -1);
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFlex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFlex.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFlex.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFlex.java Wed Nov 16 19:09:35 2011
@@ -53,7 +53,7 @@ public class TestFlex extends LuceneTest
IndexReader r = w.getReader();
- TermsEnum terms = MultiFields.getTerms(r, "field3").iterator();
+ TermsEnum terms = MultiFields.getTerms(r, "field3").iterator(null);
assertEquals(TermsEnum.SeekStatus.END, terms.seekCeil(new BytesRef("abc")));
r.close();
}
@@ -70,7 +70,7 @@ public class TestFlex extends LuceneTest
doc.add(newField("f", "a b c", TextField.TYPE_UNSTORED));
w.addDocument(doc);
IndexReader r = w.getReader();
- TermsEnum terms = r.getSequentialSubReaders()[0].fields().terms("f").iterator();
+ TermsEnum terms = r.getSequentialSubReaders()[0].fields().terms("f").iterator(null);
assertTrue(terms.next() != null);
try {
assertEquals(0, terms.ord());
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Wed Nov 16 19:09:35 2011
@@ -25,9 +25,12 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.FieldInfosReader;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MockDirectoryWrapper;
@@ -91,7 +94,8 @@ public class TestIndexFileDeleter extend
// "content", and then set our expected file names below
// accordingly:
CompoundFileDirectory cfsReader = new CompoundFileDirectory(dir, "_2.cfs", newIOContext(random), false);
- FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
+ FieldInfosReader infosReader = Codec.getDefault().fieldInfosFormat().getFieldInfosReader();
+ FieldInfos fieldInfos = infosReader.read(cfsReader, "2", IOContext.READONCE);
int contentFieldIndex = -1;
for (FieldInfo fi : fieldInfos) {
if (fi.name.equals("content")) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Wed Nov 16 19:09:35 2011
@@ -28,8 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Random;
-import java.util.Set;
-import java.util.SortedSet;
import org.junit.Assume;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryField;
@@ -314,19 +312,6 @@ public class TestIndexReader extends Luc
writer.addDocument(doc);
}
writer.close();
- IndexReader reader = IndexReader.open(d, false);
- FieldSortedTermVectorMapper mapper = new FieldSortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
- reader.getTermFreqVector(0, mapper);
- Map<String,SortedSet<TermVectorEntry>> map = mapper.getFieldToTerms();
- assertTrue("map is null and it shouldn't be", map != null);
- assertTrue("map Size: " + map.size() + " is not: " + 4, map.size() == 4);
- Set<TermVectorEntry> set = map.get("termvector");
- for (Iterator<TermVectorEntry> iterator = set.iterator(); iterator.hasNext();) {
- TermVectorEntry entry = iterator.next();
- assertTrue("entry is null and it shouldn't be", entry != null);
- if (VERBOSE) System.out.println("Entry: " + entry);
- }
- reader.close();
d.close();
}
@@ -914,8 +899,17 @@ public class TestIndexReader extends Luc
Bits liveDocs = MultiFields.getLiveDocs(index1);
while((field1=fenum1.next()) != null) {
assertEquals("Different fields", field1, fenum2.next());
- TermsEnum enum1 = fenum1.terms();
- TermsEnum enum2 = fenum2.terms();
+ Terms terms1 = fenum1.terms();
+ if (terms1 == null) {
+ assertNull(fenum2.terms());
+ continue;
+ }
+ TermsEnum enum1 = terms1.iterator(null);
+
+ Terms terms2 = fenum2.terms();
+ assertNotNull(terms2);
+ TermsEnum enum2 = terms2.iterator(null);
+
while(enum1.next() != null) {
assertEquals("Different terms", enum1.term(), enum2.next());
DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Nov 16 19:09:35 2011
@@ -25,10 +25,12 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -41,6 +43,7 @@ import org.apache.lucene.document.String
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
@@ -885,7 +888,7 @@ public class TestIndexWriter extends Luc
writer.close();
IndexReader reader = IndexReader.open(dir, true);
IndexReader subreader = getOnlySegmentReader(reader);
- TermsEnum te = subreader.fields().terms("").iterator();
+ TermsEnum te = subreader.fields().terms("").iterator(null);
assertEquals(new BytesRef("a"), te.next());
assertEquals(new BytesRef("b"), te.next());
assertEquals(new BytesRef("c"), te.next());
@@ -906,7 +909,7 @@ public class TestIndexWriter extends Luc
writer.close();
IndexReader reader = IndexReader.open(dir, true);
IndexReader subreader = getOnlySegmentReader(reader);
- TermsEnum te = subreader.fields().terms("").iterator();
+ TermsEnum te = subreader.fields().terms("").iterator(null);
assertEquals(new BytesRef(""), te.next());
assertEquals(new BytesRef("a"), te.next());
assertEquals(new BytesRef("b"), te.next());
@@ -1071,13 +1074,23 @@ public class TestIndexWriter extends Luc
w.close();
IndexReader r = IndexReader.open(dir, true);
- TermPositionVector tpv = ((TermPositionVector) r.getTermFreqVector(0, "field"));
- int[] poss = tpv.getTermPositions(0);
- assertEquals(1, poss.length);
- assertEquals(100, poss[0]);
- poss = tpv.getTermPositions(1);
- assertEquals(1, poss.length);
- assertEquals(101, poss[0]);
+ Terms tpv = r.getTermVectors(0).terms("field");
+ TermsEnum termsEnum = tpv.iterator(null);
+ assertNotNull(termsEnum.next());
+ DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+ assertNotNull(dpEnum);
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(1, dpEnum.freq());
+ assertEquals(100, dpEnum.nextPosition());
+
+ assertNotNull(termsEnum.next());
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ assertNotNull(dpEnum);
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(1, dpEnum.freq());
+ assertEquals(101, dpEnum.nextPosition());
+ assertNull(termsEnum.next());
+
r.close();
dir.close();
}
@@ -1402,7 +1415,7 @@ public class TestIndexWriter extends Luc
w.addDocument(d);
IndexReader r = w.getReader().getSequentialSubReaders()[0];
- TermsEnum t = r.fields().terms("field").iterator();
+ TermsEnum t = r.fields().terms("field").iterator(null);
int count = 0;
while(t.next() != null) {
final DocsEnum docs = t.docs(null, null);
@@ -1740,13 +1753,16 @@ public class TestIndexWriter extends Luc
_TestUtil.checkIndex(dir);
assertNoUnreferencedFiles(dir, "no tv files");
- String[] files = dir.listAll();
- for(String file : files) {
- assertTrue(!file.endsWith(IndexFileNames.VECTORS_FIELDS_EXTENSION));
- assertTrue(!file.endsWith(IndexFileNames.VECTORS_INDEX_EXTENSION));
- assertTrue(!file.endsWith(IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
+ IndexReader r0 = IndexReader.open(dir);
+ for (IndexReader r : r0.getSequentialSubReaders()) {
+ SegmentInfo s = ((SegmentReader) r).getSegmentInfo();
+ assertFalse(s.getHasVectors());
+ Set<String> files = new HashSet<String>();
+ s.getCodec().termVectorsFormat().files(dir, s, files);
+ assertTrue(files.isEmpty());
}
-
+
+ r0.close();
dir.close();
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Wed Nov 16 19:09:35 2011
@@ -535,7 +535,7 @@ public class TestIndexWriterExceptions e
boolean sawAppend = false;
boolean sawFlush = false;
for (int i = 0; i < trace.length; i++) {
- if ("org.apache.lucene.index.FreqProxTermsWriterPerField".equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName()))
+ if (FreqProxTermsWriterPerField.class.getName().equals(trace[i].getClassName()) && "flush".equals(trace[i].getMethodName()))
sawAppend = true;
if ("flush".equals(trace[i].getMethodName()))
sawFlush = true;
@@ -641,7 +641,7 @@ public class TestIndexWriterExceptions e
numDel++;
else {
reader.document(j);
- reader.getTermFreqVectors(j);
+ reader.getTermVectors(j);
}
}
assertEquals(1, numDel);
@@ -665,7 +665,7 @@ public class TestIndexWriterExceptions e
assertNull(MultiFields.getLiveDocs(reader));
for(int j=0;j<reader.maxDoc();j++) {
reader.document(j);
- reader.getTermFreqVectors(j);
+ reader.getTermVectors(j);
}
reader.close();
assertEquals(0, numDel);
@@ -755,7 +755,7 @@ public class TestIndexWriterExceptions e
numDel++;
else {
reader.document(j);
- reader.getTermFreqVectors(j);
+ reader.getTermVectors(j);
}
}
reader.close();
@@ -778,7 +778,7 @@ public class TestIndexWriterExceptions e
assertNull(MultiFields.getLiveDocs(reader));
for(int j=0;j<reader.maxDoc();j++) {
reader.document(j);
- reader.getTermFreqVectors(j);
+ reader.getTermVectors(j);
}
reader.close();
@@ -794,7 +794,7 @@ public class TestIndexWriterExceptions e
if (doFail) {
StackTraceElement[] trace = new Exception().getStackTrace();
for (int i = 0; i < trace.length; i++) {
- if (doFail && "org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "sync".equals(trace[i].getMethodName())) {
+ if (doFail && MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "sync".equals(trace[i].getMethodName())) {
didFail = true;
throw new IOException("now failing on purpose during sync");
}
@@ -868,11 +868,11 @@ public class TestIndexWriterExceptions e
boolean isDelete = false;
boolean isInGlobalFieldMap = false;
for (int i = 0; i < trace.length; i++) {
- if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName()))
+ if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName()))
isCommit = true;
- if ("org.apache.lucene.store.MockDirectoryWrapper".equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
+ if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName()))
isDelete = true;
- if ("org.apache.lucene.index.SegmentInfos".equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName()))
+ if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName()))
isInGlobalFieldMap = true;
}
@@ -901,6 +901,7 @@ public class TestIndexWriterExceptions e
for (FailOnlyInCommit failure : failures) {
MockDirectoryWrapper dir = newDirectory();
+ dir.setFailOnCreateOutput(false);
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)));
Document doc = new Document();
@@ -1302,20 +1303,17 @@ public class TestIndexWriterExceptions e
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
+
StackTraceElement[] trace = new Exception().getStackTrace();
- boolean failOnInit = false;
- boolean failOnfinish = false;
+ boolean fail = false;
for (int i = 0; i < trace.length; i++) {
- if ("org.apache.lucene.index.TermVectorsTermsWriter".equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName()))
- failOnInit = true;
- if ("org.apache.lucene.index.TermVectorsTermsWriter".equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName()))
- failOnfinish = true;
+ if (TermVectorsConsumer.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) {
+ fail = true;
+ }
}
- if (failOnInit) {
- throw new RuntimeException(EXC_MSG + " fail on init");
- } else if (failOnfinish) {
- throw new RuntimeException(EXC_MSG + " fail on finishDoc");
+ if (fail) {
+ throw new RuntimeException(EXC_MSG);
}
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java Wed Nov 16 19:09:35 2011
@@ -135,7 +135,7 @@ public class TestIndexWriterUnicode exte
}
private void checkTermsOrder(IndexReader r, Set<String> allTerms, boolean isTop) throws IOException {
- TermsEnum terms = MultiFields.getFields(r).terms("f").iterator();
+ TermsEnum terms = MultiFields.getFields(r).terms("f").iterator(null);
BytesRef last = new BytesRef();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Wed Nov 16 19:09:35 2011
@@ -278,7 +278,7 @@ public class TestIndexWriterWithThreads
for(int j=0;j<reader.maxDoc();j++) {
if (delDocs == null || !delDocs.get(j)) {
reader.document(j);
- reader.getTermFreqVectors(j);
+ reader.getTermVectors(j);
}
}
reader.close();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java Wed Nov 16 19:09:35 2011
@@ -288,33 +288,31 @@ public class TestIndexableField extends
if (indexed) {
final boolean tv = counter % 2 == 1 && fieldID != 9;
- final TermFreqVector tfv = r.getTermFreqVector(docID, name);
if (tv) {
+ final Terms tfv = r.getTermVectors(docID).terms(name);
assertNotNull(tfv);
- assertTrue(tfv instanceof TermPositionVector);
- final TermPositionVector tpv = (TermPositionVector) tfv;
- final BytesRef[] terms = tpv.getTerms();
- assertEquals(2, terms.length);
- assertEquals(new BytesRef(""+counter), terms[0]);
- assertEquals(new BytesRef("text"), terms[1]);
-
- final int[] freqs = tpv.getTermFrequencies();
- assertEquals(2, freqs.length);
- assertEquals(1, freqs[0]);
- assertEquals(1, freqs[1]);
-
- int[] positions = tpv.getTermPositions(0);
- assertEquals(1, positions.length);
- assertEquals(1, positions[0]);
-
- positions = tpv.getTermPositions(1);
- assertEquals(1, positions.length);
- assertEquals(0, positions[0]);
+ TermsEnum termsEnum = tfv.iterator(null);
+ assertEquals(new BytesRef(""+counter), termsEnum.next());
+ assertEquals(1, termsEnum.totalTermFreq());
+ DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(1, dpEnum.freq());
+ assertEquals(1, dpEnum.nextPosition());
+
+ assertEquals(new BytesRef("text"), termsEnum.next());
+ assertEquals(1, termsEnum.totalTermFreq());
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(1, dpEnum.freq());
+ assertEquals(0, dpEnum.nextPosition());
+
+ assertNull(termsEnum.next());
// TODO: offsets
} else {
- assertNull(tfv);
+ Fields vectors = r.getTermVectors(docID);
+ assertTrue(vectors == null || vectors.terms(name) == null);
}
if (numeric) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java Wed Nov 16 19:09:35 2011
@@ -84,7 +84,8 @@ public class TestParallelTermEnum extend
f = fe.next();
assertEquals("field1", f);
- TermsEnum te = fe.terms();
+ Terms terms = fe.terms();
+ TermsEnum te = terms.iterator(null);
assertEquals("brown", te.next().utf8ToString());
DocsEnum td = te.docs(liveDocs, null);
@@ -119,7 +120,9 @@ public class TestParallelTermEnum extend
assertNull(te.next());
f = fe.next();
assertEquals("field2", f);
- te = fe.terms();
+ terms = fe.terms();
+ assertNotNull(terms);
+ te = terms.iterator(null);
assertEquals("brown", te.next().utf8ToString());
td = te.docs(liveDocs, td);
@@ -154,7 +157,9 @@ public class TestParallelTermEnum extend
assertNull(te.next());
f = fe.next();
assertEquals("field3", f);
- te = fe.terms();
+ terms = fe.terms();
+ assertNotNull(terms);
+ te = terms.iterator(null);
assertEquals("dog", te.next().utf8ToString());
td = te.docs(liveDocs, td);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java Wed Nov 16 19:09:35 2011
@@ -527,7 +527,7 @@ public class TestPayloads extends Lucene
}
writer.close();
IndexReader reader = IndexReader.open(dir, true);
- TermsEnum terms = MultiFields.getFields(reader).terms(field).iterator();
+ TermsEnum terms = MultiFields.getFields(reader).terms(field).iterator(null);
Bits liveDocs = MultiFields.getLiveDocs(reader);
DocsAndPositionsEnum tp = null;
while (terms.next() != null) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java Wed Nov 16 19:09:35 2011
@@ -222,7 +222,7 @@ public class TestPerSegmentDeletes exten
throws IOException {
Fields fields = MultiFields.getFields(reader);
Terms cterms = fields.terms(term.field);
- TermsEnum ctermsEnum = cterms.iterator();
+ TermsEnum ctermsEnum = cterms.iterator(null);
if (ctermsEnum.seekExact(new BytesRef(term.text()), false)) {
DocsEnum docsEnum = ctermsEnum.docs(bits, null);
return toArray(docsEnum);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Wed Nov 16 19:09:35 2011
@@ -77,7 +77,7 @@ public class TestSegmentMerger extends L
public void testMerge() throws IOException {
final Codec codec = Codec.getDefault();
- SegmentMerger merger = new SegmentMerger(InfoStream.getDefault(), mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, MergeState.CheckAbort.NONE, null, new FieldInfos(), codec, newIOContext(random));
+ SegmentMerger merger = new SegmentMerger(InfoStream.getDefault(), mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, MergeState.CheckAbort.NONE, null, new FieldInfos(new FieldInfos.FieldNumberBiMap()), codec, newIOContext(random));
merger.add(reader1);
merger.add(reader2);
MergeState mergeState = merger.merge();
@@ -110,23 +110,19 @@ public class TestSegmentMerger extends L
//System.out.println("stored size: " + stored.size());
assertTrue("We do not have 3 fields that were indexed with term vector",stored.size() == 3);
- TermFreqVector vector = mergedReader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);
- assertTrue(vector != null);
- BytesRef [] terms = vector.getTerms();
- assertTrue(terms != null);
- //System.out.println("Terms size: " + terms.length);
- assertTrue(terms.length == 3);
- int [] freqs = vector.getTermFrequencies();
- assertTrue(freqs != null);
- //System.out.println("Freqs size: " + freqs.length);
- assertTrue(vector instanceof TermPositionVector == true);
-
- for (int i = 0; i < terms.length; i++) {
- String term = terms[i].utf8ToString();
- int freq = freqs[i];
+ Terms vector = mergedReader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
+ assertNotNull(vector);
+ assertEquals(3, vector.getUniqueTermCount());
+ TermsEnum termsEnum = vector.iterator(null);
+
+ int i = 0;
+ while (termsEnum.next() != null) {
+ String term = termsEnum.term().utf8ToString();
+ int freq = (int) termsEnum.totalTermFreq();
//System.out.println("Term: " + term + " Freq: " + freq);
assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);
assertTrue(DocHelper.FIELD_2_FREQS[i] == freq);
+ i++;
}
TestSegmentReader.checkNorms(mergedReader);
@@ -152,7 +148,7 @@ public class TestSegmentMerger extends L
SegmentMerger sm = new SegmentMerger(InfoStream.getDefault(), dir, 1, "a", MergeState.CheckAbort.NONE, null, null, Codec.getDefault(), newIOContext(random));
boolean doFail = false;
try {
- sm.createCompoundFile("b1", w.segmentInfos.info(0), newIOContext(random));
+ IndexWriter.createCompoundFile(dir, "b1", MergeState.CheckAbort.NONE, w.segmentInfos.info(0), newIOContext(random));
doFail = true; // should never get here
} catch (AssertionError e) {
// expected
@@ -173,7 +169,7 @@ public class TestSegmentMerger extends L
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
try {
- sm.createCompoundFile("b2", sis.info(0), newIOContext(random));
+ IndexWriter.createCompoundFile(dir, "b2", MergeState.CheckAbort.NONE, sis.info(0), newIOContext(random));
doFail = true; // should never get here
} catch (AssertionError e) {
// expected
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java Wed Nov 16 19:09:35 2011
@@ -121,9 +121,11 @@ public class TestSegmentReader extends L
FieldsEnum fields = MultiFields.getFields(reader).iterator();
String field;
while((field = fields.next()) != null) {
- TermsEnum terms = fields.terms();
- while(terms.next() != null) {
- BytesRef term = terms.term();
+ Terms terms = fields.terms();
+ assertNotNull(terms);
+ TermsEnum termsEnum = terms.iterator(null);
+ while(termsEnum.next() != null) {
+ BytesRef term = termsEnum.term();
assertTrue(term != null);
String fieldValue = (String) DocHelper.nameValues.get(field);
assertTrue(fieldValue.indexOf(term.utf8ToString()) != -1);
@@ -188,20 +190,19 @@ public class TestSegmentReader extends L
}
public void testTermVectors() throws IOException {
- TermFreqVector result = reader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);
- assertTrue(result != null);
- BytesRef [] terms = result.getTerms();
- int [] freqs = result.getTermFrequencies();
- assertTrue(terms != null && terms.length == 3 && freqs != null && freqs.length == 3);
- for (int i = 0; i < terms.length; i++) {
- String term = terms[i].utf8ToString();
- int freq = freqs[i];
+ Terms result = reader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
+ assertNotNull(result);
+ assertEquals(3, result.getUniqueTermCount());
+ TermsEnum termsEnum = result.iterator(null);
+ while(termsEnum.next() != null) {
+ String term = termsEnum.term().utf8ToString();
+ int freq = (int) termsEnum.totalTermFreq();
assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);
assertTrue(freq > 0);
}
- TermFreqVector [] results = reader.getTermFreqVectors(0);
+ Fields results = reader.getTermVectors(0);
assertTrue(results != null);
- assertTrue("We do not have 3 term freq vectors, we have: " + results.length, results.length == 3);
+ assertEquals("We do not have 3 term freq vectors", 3, results.getUniqueFieldCount());
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java Wed Nov 16 19:09:35 2011
@@ -59,7 +59,7 @@ public class TestSegmentTermDocs extends
assertTrue(reader != null);
assertEquals(indexDivisor, reader.getTermInfosIndexDivisor());
- TermsEnum terms = reader.fields().terms(DocHelper.TEXT_FIELD_2_KEY).iterator();
+ TermsEnum terms = reader.fields().terms(DocHelper.TEXT_FIELD_2_KEY).iterator(null);
terms.seekCeil(new BytesRef("field"));
DocsEnum termDocs = terms.docs(reader.getLiveDocs(), null);
if (termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java Wed Nov 16 19:09:35 2011
@@ -79,7 +79,7 @@ public class TestSegmentTermEnum extends
addDoc(writer, "aaa bbb");
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir, false));
- TermsEnum terms = reader.fields().terms("content").iterator();
+ TermsEnum terms = reader.fields().terms("content").iterator(null);
assertNotNull(terms.next());
assertEquals("aaa", terms.term().utf8ToString());
assertNotNull(terms.next());
@@ -103,7 +103,7 @@ public class TestSegmentTermEnum extends
throws IOException
{
IndexReader reader = IndexReader.open(dir, true);
- TermsEnum termEnum = MultiFields.getTerms(reader, "content").iterator();
+ TermsEnum termEnum = MultiFields.getTerms(reader, "content").iterator(null);
// create enumeration of all terms
// go to the first term (aaa)
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSizeBoundedForceMerge.java Wed Nov 16 19:09:35 2011
@@ -59,7 +59,7 @@ public class TestSizeBoundedForceMerge e
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
- double min = sis.info(0).sizeInBytes(true);
+ double min = sis.info(0).sizeInBytes();
conf = newWriterConfig();
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java Wed Nov 16 19:09:35 2011
@@ -67,7 +67,7 @@ public class TestStressAdvance extends L
bDocIDs.add(docID);
}
}
- final TermsEnum te = r.getSequentialSubReaders()[0].fields().terms("field").iterator();
+ final TermsEnum te = r.getSequentialSubReaders()[0].fields().terms("field").iterator(null);
DocsEnum de = null;
for(int iter2=0;iter2<10;iter2++) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java Wed Nov 16 19:09:35 2011
@@ -24,11 +24,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import org.apache.lucene.util.*;
import junit.framework.Assert;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -36,6 +36,7 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.*;
public class TestStressIndexing2 extends LuceneTestCase {
static int maxFields=4;
@@ -322,7 +323,7 @@ public class TestStressIndexing2 extends
MultiFields.getFields(r2).terms(idField) == null);
return;
}
- final TermsEnum termsEnum = terms1.iterator();
+ final TermsEnum termsEnum = terms1.iterator(null);
final Bits liveDocs1 = MultiFields.getLiveDocs(r1);
final Bits liveDocs2 = MultiFields.getLiveDocs(r2);
@@ -382,24 +383,79 @@ public class TestStressIndexing2 extends
try {
// verify term vectors are equivalent
- verifyEquals(r1.getTermFreqVectors(id1), r2.getTermFreqVectors(id2));
+ verifyEquals(r1.getTermVectors(id1), r2.getTermVectors(id2));
} catch (Throwable e) {
System.out.println("FAILED id=" + term + " id1=" + id1 + " id2=" + id2);
- TermFreqVector[] tv1 = r1.getTermFreqVectors(id1);
+ Fields tv1 = r1.getTermVectors(id1);
System.out.println(" d1=" + tv1);
- if (tv1 != null)
- for(int i=0;i<tv1.length;i++)
- System.out.println(" " + i + ": " + tv1[i]);
+ if (tv1 != null) {
+ FieldsEnum fieldsEnum = tv1.iterator();
+ String field;
+ DocsAndPositionsEnum dpEnum = null;
+ DocsEnum dEnum = null;
+ while ((field=fieldsEnum.next()) != null) {
+ System.out.println(" " + field + ":");
+ Terms terms3 = fieldsEnum.terms();
+ assertNotNull(terms3);
+ TermsEnum termsEnum2 = terms3.iterator(null);
+ BytesRef term2;
+ while((term2 = termsEnum2.next()) != null) {
+ System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum2.totalTermFreq());
+ dpEnum = termsEnum2.docsAndPositions(null, dpEnum);
+ if (dpEnum != null) {
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ final int freq = dpEnum.freq();
+ System.out.println(" doc=" + dpEnum.docID() + " freq=" + freq);
+ for(int posUpto=0;posUpto<freq;posUpto++) {
+ System.out.println(" pos=" + dpEnum.nextPosition());
+ }
+ } else {
+ dEnum = termsEnum2.docs(null, dEnum);
+ assertNotNull(dEnum);
+ assertTrue(dEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ final int freq = dEnum.freq();
+ System.out.println(" doc=" + dEnum.docID() + " freq=" + freq);
+ }
+ }
+ }
+ }
- TermFreqVector[] tv2 = r2.getTermFreqVectors(id2);
+ Fields tv2 = r2.getTermVectors(id2);
System.out.println(" d2=" + tv2);
- if (tv2 != null)
- for(int i=0;i<tv2.length;i++)
- System.out.println(" " + i + ": " + tv2[i]);
+ if (tv2 != null) {
+ FieldsEnum fieldsEnum = tv2.iterator();
+ String field;
+ DocsAndPositionsEnum dpEnum = null;
+ DocsEnum dEnum = null;
+ while ((field=fieldsEnum.next()) != null) {
+ System.out.println(" " + field + ":");
+ Terms terms3 = fieldsEnum.terms();
+ assertNotNull(terms3);
+ TermsEnum termsEnum2 = terms3.iterator(null);
+ BytesRef term2;
+ while((term2 = termsEnum2.next()) != null) {
+ System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum2.totalTermFreq());
+ dpEnum = termsEnum2.docsAndPositions(null, dpEnum);
+ if (dpEnum != null) {
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ final int freq = dpEnum.freq();
+ System.out.println(" doc=" + dpEnum.docID() + " freq=" + freq);
+ for(int posUpto=0;posUpto<freq;posUpto++) {
+ System.out.println(" pos=" + dpEnum.nextPosition());
+ }
+ } else {
+ dEnum = termsEnum2.docs(null, dEnum);
+ assertNotNull(dEnum);
+ assertTrue(dEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ final int freq = dEnum.freq();
+ System.out.println(" doc=" + dEnum.docID() + " freq=" + freq);
+ }
+ }
+ }
+ }
throw e;
}
-
}
//System.out.println("TEST: done match id");
@@ -430,7 +486,11 @@ public class TestStressIndexing2 extends
if (field1 == null) {
break;
} else {
- termsEnum1 = fields1.terms();
+ Terms terms = fields1.terms();
+ if (terms == null) {
+ continue;
+ }
+ termsEnum1 = terms.iterator(null);
}
}
term1 = termsEnum1.next();
@@ -460,7 +520,11 @@ public class TestStressIndexing2 extends
if (field2 == null) {
break;
} else {
- termsEnum2 = fields2.terms();
+ Terms terms = fields2.terms();
+ if (terms == null) {
+ continue;
+ }
+ termsEnum2 = terms.iterator(null);
}
}
term2 = termsEnum2.next();
@@ -527,62 +591,98 @@ public class TestStressIndexing2 extends
}
}
- public static void verifyEquals(TermFreqVector[] d1, TermFreqVector[] d2) {
+ public static void verifyEquals(Fields d1, Fields d2) throws IOException {
if (d1 == null) {
- assertTrue(d2 == null);
+ assertTrue(d2 == null || d2.getUniqueFieldCount() == 0);
return;
}
assertTrue(d2 != null);
- assertEquals(d1.length, d2.length);
- for(int i=0;i<d1.length;i++) {
- TermFreqVector v1 = d1[i];
- TermFreqVector v2 = d2[i];
- if (v1 == null || v2 == null)
- System.out.println("v1=" + v1 + " v2=" + v2 + " i=" + i + " of " + d1.length);
- assertEquals(v1.size(), v2.size());
- int numTerms = v1.size();
- BytesRef[] terms1 = v1.getTerms();
- BytesRef[] terms2 = v2.getTerms();
- int[] freq1 = v1.getTermFrequencies();
- int[] freq2 = v2.getTermFrequencies();
- for(int j=0;j<numTerms;j++) {
- if (!terms1[j].equals(terms2[j]))
- assertEquals(terms1[j], terms2[j]);
- assertEquals(freq1[j], freq2[j]);
- }
- if (v1 instanceof TermPositionVector) {
- assertTrue(v2 instanceof TermPositionVector);
- TermPositionVector tpv1 = (TermPositionVector) v1;
- TermPositionVector tpv2 = (TermPositionVector) v2;
- for(int j=0;j<numTerms;j++) {
- int[] pos1 = tpv1.getTermPositions(j);
- int[] pos2 = tpv2.getTermPositions(j);
- if (pos1 == null) {
- assertNull(pos2);
+ FieldsEnum fieldsEnum1 = d1.iterator();
+ FieldsEnum fieldsEnum2 = d2.iterator();
+ String field1;
+ while ((field1 = fieldsEnum1.next()) != null) {
+ String field2 = fieldsEnum2.next();
+ assertEquals(field1, field2);
+
+ Terms terms1 = fieldsEnum1.terms();
+ assertNotNull(terms1);
+ TermsEnum termsEnum1 = terms1.iterator(null);
+
+ Terms terms2 = fieldsEnum2.terms();
+ assertNotNull(terms2);
+ TermsEnum termsEnum2 = terms2.iterator(null);
+
+ DocsAndPositionsEnum dpEnum1 = null;
+ DocsAndPositionsEnum dpEnum2 = null;
+ DocsEnum dEnum1 = null;
+ DocsEnum dEnum2 = null;
+
+ BytesRef term1;
+ while ((term1 = termsEnum1.next()) != null) {
+ BytesRef term2 = termsEnum2.next();
+ assertEquals(term1, term2);
+ assertEquals(termsEnum1.totalTermFreq(),
+ termsEnum2.totalTermFreq());
+
+ dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1);
+ dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2);
+ if (dpEnum1 != null) {
+ assertNotNull(dpEnum2);
+ int docID1 = dpEnum1.nextDoc();
+ dpEnum2.nextDoc();
+ // docIDs are not supposed to be equal
+ //int docID2 = dpEnum2.nextDoc();
+ //assertEquals(docID1, docID2);
+ assertTrue(docID1 != DocsEnum.NO_MORE_DOCS);
+
+ int freq1 = dpEnum1.freq();
+ int freq2 = dpEnum2.freq();
+ assertEquals(freq1, freq2);
+ OffsetAttribute offsetAtt1 = dpEnum1.attributes().hasAttribute(OffsetAttribute.class) ? dpEnum1.attributes().getAttribute(OffsetAttribute.class) : null;
+ OffsetAttribute offsetAtt2 = dpEnum2.attributes().hasAttribute(OffsetAttribute.class) ? dpEnum2.attributes().getAttribute(OffsetAttribute.class) : null;
+
+ if (offsetAtt1 != null) {
+ assertNotNull(offsetAtt2);
} else {
- assertNotNull(pos1);
- assertNotNull(pos2);
- assertEquals(pos1.length, pos2.length);
- TermVectorOffsetInfo[] offsets1 = tpv1.getOffsets(j);
- TermVectorOffsetInfo[] offsets2 = tpv2.getOffsets(j);
- if (offsets1 == null)
- assertTrue(offsets2 == null);
- else
- assertTrue(offsets2 != null);
- for(int k=0;k<pos1.length;k++) {
- assertEquals(pos1[k], pos2[k]);
- if (offsets1 != null) {
- assertEquals(offsets1[k].getStartOffset(),
- offsets2[k].getStartOffset());
- assertEquals(offsets1[k].getEndOffset(),
- offsets2[k].getEndOffset());
- }
+ assertNull(offsetAtt2);
+ }
+
+ for(int posUpto=0;posUpto<freq1;posUpto++) {
+ int pos1 = dpEnum1.nextPosition();
+ int pos2 = dpEnum2.nextPosition();
+ assertEquals(pos1, pos2);
+ if (offsetAtt1 != null) {
+ assertEquals(offsetAtt1.startOffset(),
+ offsetAtt2.startOffset());
+ assertEquals(offsetAtt1.endOffset(),
+ offsetAtt2.endOffset());
}
}
+ assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum1.nextDoc());
+ assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum2.nextDoc());
+ } else {
+ dEnum1 = termsEnum1.docs(null, dEnum1);
+ dEnum2 = termsEnum2.docs(null, dEnum2);
+ assertNotNull(dEnum1);
+ assertNotNull(dEnum2);
+ int docID1 = dEnum1.nextDoc();
+ dEnum2.nextDoc();
+ // docIDs are not supposed to be equal
+ //int docID2 = dEnum2.nextDoc();
+ //assertEquals(docID1, docID2);
+ assertTrue(docID1 != DocsEnum.NO_MORE_DOCS);
+ int freq1 = dEnum1.freq();
+ int freq2 = dEnum2.freq();
+ assertEquals(freq1, freq2);
+ assertEquals(DocsEnum.NO_MORE_DOCS, dEnum1.nextDoc());
+ assertEquals(DocsEnum.NO_MORE_DOCS, dEnum2.nextDoc());
}
}
+
+ assertNull(termsEnum2.next());
}
+ assertNull(fieldsEnum2.next());
}
private class IndexingThread extends Thread {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSumDocFreq.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSumDocFreq.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSumDocFreq.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSumDocFreq.java Wed Nov 16 19:09:35 2011
@@ -92,7 +92,7 @@ public class TestSumDocFreq extends Luce
}
long computedSumDocFreq = 0;
- TermsEnum termsEnum = terms.iterator();
+ TermsEnum termsEnum = terms.iterator(null);
while (termsEnum.next() != null) {
computedSumDocFreq += termsEnum.docFreq();
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1202842&r1=1202841&r2=1202842&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Wed Nov 16 19:09:35 2011
@@ -20,9 +20,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
import java.util.Arrays;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.SortedSet;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@@ -32,6 +31,9 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.DefaultTermVectorsReader;
+import org.apache.lucene.index.codecs.TermVectorsReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -43,10 +45,9 @@ public class TestTermVectorsReader exten
private boolean[] testFieldsStoreOff = {true, false, false, true};
private String[] testTerms = {"this", "is", "a", "test"};
private int[][] positions = new int[testTerms.length][];
- private TermVectorOffsetInfo[][] offsets = new TermVectorOffsetInfo[testTerms.length][];
private Directory dir;
- private String seg;
- private FieldInfos fieldInfos = new FieldInfos();
+ private SegmentInfo seg;
+ private FieldInfos fieldInfos = new FieldInfos(new FieldInfos.FieldNumberBiMap());
private static int TERM_FREQ = 3;
private class TestToken implements Comparable<TestToken> {
@@ -74,18 +75,15 @@ public class TestTermVectorsReader exten
int tokenUpto = 0;
for (int i = 0; i < testTerms.length; i++) {
positions[i] = new int[TERM_FREQ];
- offsets[i] = new TermVectorOffsetInfo[TERM_FREQ];
// first position must be 0
for (int j = 0; j < TERM_FREQ; j++) {
// positions are always sorted in increasing order
positions[i][j] = (int) (j * 10 + Math.random() * 10);
- // offsets are always sorted in increasing order
- offsets[i][j] = new TermVectorOffsetInfo(j * 10, j * 10 + testTerms[i].length());
TestToken token = tokens[tokenUpto++] = new TestToken();
token.text = testTerms[i];
token.pos = positions[i][j];
- token.startOffset = offsets[i][j].getStartOffset();
- token.endOffset = offsets[i][j].getEndOffset();
+ token.startOffset = j * 10;
+ token.endOffset = j * 10 + testTerms[i].length();
}
}
Arrays.sort(tokens);
@@ -122,13 +120,14 @@ public class TestTermVectorsReader exten
//Create 5 documents for testing, they all have the same
//terms
- for(int j=0;j<5;j++)
+ for(int j=0;j<5;j++) {
writer.addDocument(doc);
+ }
writer.commit();
- seg = writer.newestSegment().name;
+ seg = writer.newestSegment();
writer.close();
- fieldInfos = new FieldInfos(dir, IndexFileNames.segmentFileName(seg, "", IndexFileNames.FIELD_INFOS_EXTENSION));
+ fieldInfos = seg.getFieldInfos(); //new FieldInfos(dir, IndexFileNames.segmentFileName(seg.name, "", IndexFileNames.FIELD_INFOS_EXTENSION));
}
@Override
@@ -152,9 +151,9 @@ public class TestTermVectorsReader exten
@Override
public boolean incrementToken() {
- if (tokenUpto >= tokens.length)
+ if (tokenUpto >= tokens.length) {
return false;
- else {
+ } else {
final TestToken testToken = tokens[tokenUpto++];
clearAttributes();
termAtt.append(testToken.text);
@@ -184,282 +183,150 @@ public class TestTermVectorsReader exten
public void test() throws IOException {
//Check to see the files were created properly in setup
- assertTrue(dir.fileExists(IndexFileNames.segmentFileName(seg, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)));
- assertTrue(dir.fileExists(IndexFileNames.segmentFileName(seg, "", IndexFileNames.VECTORS_INDEX_EXTENSION)));
+ IndexReader reader = IndexReader.open(dir);
+ for (IndexReader r : reader.getSequentialSubReaders()) {
+ SegmentInfo s = ((SegmentReader) r).getSegmentInfo();
+ assertTrue(s.getHasVectors());
+ Set<String> files = new HashSet<String>();
+ s.getCodec().termVectorsFormat().files(dir, s, files);
+ assertFalse(files.isEmpty());
+ for (String file : files) {
+ assertTrue(dir.fileExists(file));
+ }
+ }
+ reader.close();
}
public void testReader() throws IOException {
- TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ TermVectorsReader reader = Codec.getDefault().termVectorsFormat().vectorsReader(dir, seg, fieldInfos, newIOContext(random));
for (int j = 0; j < 5; j++) {
- TermFreqVector vector = reader.get(j, testFields[0]);
- assertTrue(vector != null);
- BytesRef[] terms = vector.getTerms();
- assertTrue(terms != null);
- assertTrue(terms.length == testTerms.length);
- for (int i = 0; i < terms.length; i++) {
- String term = terms[i].utf8ToString();
+ Terms vector = reader.get(j).terms(testFields[0]);
+ assertNotNull(vector);
+ assertEquals(testTerms.length, vector.getUniqueTermCount());
+ TermsEnum termsEnum = vector.iterator(null);
+ for (int i = 0; i < testTerms.length; i++) {
+ final BytesRef text = termsEnum.next();
+ assertNotNull(text);
+ String term = text.utf8ToString();
//System.out.println("Term: " + term);
- assertTrue(term.equals(testTerms[i]));
+ assertEquals(testTerms[i], term);
}
+ assertNull(termsEnum.next());
}
reader.close();
}
public void testPositionReader() throws IOException {
- TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
- TermPositionVector vector;
+ TermVectorsReader reader = Codec.getDefault().termVectorsFormat().vectorsReader(dir, seg, fieldInfos, newIOContext(random));
BytesRef[] terms;
- vector = (TermPositionVector) reader.get(0, testFields[0]);
- assertTrue(vector != null);
- terms = vector.getTerms();
- assertTrue(terms != null);
- assertTrue(terms.length == testTerms.length);
- for (int i = 0; i < terms.length; i++) {
- String term = terms[i].utf8ToString();
+ Terms vector = reader.get(0).terms(testFields[0]);
+ assertNotNull(vector);
+ assertEquals(testTerms.length, vector.getUniqueTermCount());
+ TermsEnum termsEnum = vector.iterator(null);
+ DocsAndPositionsEnum dpEnum = null;
+ for (int i = 0; i < testTerms.length; i++) {
+ final BytesRef text = termsEnum.next();
+ assertNotNull(text);
+ String term = text.utf8ToString();
//System.out.println("Term: " + term);
- assertTrue(term.equals(testTerms[i]));
- int[] positions = vector.getTermPositions(i);
- assertTrue(positions != null);
- assertTrue(positions.length == this.positions[i].length);
- for (int j = 0; j < positions.length; j++) {
- int position = positions[j];
- assertTrue(position == this.positions[i][j]);
- }
- TermVectorOffsetInfo[] offset = vector.getOffsets(i);
- assertTrue(offset != null);
- assertTrue(offset.length == this.offsets[i].length);
- for (int j = 0; j < offset.length; j++) {
- TermVectorOffsetInfo termVectorOffsetInfo = offset[j];
- assertTrue(termVectorOffsetInfo.equals(offsets[i][j]));
- }
- }
+ assertEquals(testTerms[i], term);
- TermFreqVector freqVector = reader.get(0, testFields[1]); //no pos, no offset
- assertTrue(freqVector != null);
- assertTrue(freqVector instanceof TermPositionVector == false);
- terms = freqVector.getTerms();
- assertTrue(terms != null);
- assertTrue(terms.length == testTerms.length);
- for (int i = 0; i < terms.length; i++) {
- String term = terms[i].utf8ToString();
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ assertNotNull(dpEnum);
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(dpEnum.freq(), positions[i].length);
+ for (int j = 0; j < positions[i].length; j++) {
+ assertEquals(positions[i][j], dpEnum.nextPosition());
+ }
+ assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
+
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertNotNull(dpEnum);
+ final OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
+ assertNotNull(offsetAtt);
+ assertEquals(dpEnum.freq(), positions[i].length);
+ for (int j = 0; j < positions[i].length; j++) {
+ assertEquals(positions[i][j], dpEnum.nextPosition());
+ assertEquals(j*10, offsetAtt.startOffset());
+ assertEquals(j*10 + testTerms[i].length(), offsetAtt.endOffset());
+ }
+ assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
+ }
+
+ Terms freqVector = reader.get(0).terms(testFields[1]); //no pos, no offset
+ assertNotNull(freqVector);
+ assertEquals(testTerms.length, freqVector.getUniqueTermCount());
+ termsEnum = freqVector.iterator(null);
+ assertNotNull(termsEnum);
+ for (int i = 0; i < testTerms.length; i++) {
+ final BytesRef text = termsEnum.next();
+ assertNotNull(text);
+ String term = text.utf8ToString();
//System.out.println("Term: " + term);
- assertTrue(term.equals(testTerms[i]));
+ assertEquals(testTerms[i], term);
}
reader.close();
}
public void testOffsetReader() throws IOException {
- TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
- TermPositionVector vector = (TermPositionVector) reader.get(0, testFields[0]);
- assertTrue(vector != null);
- BytesRef[] terms = vector.getTerms();
- assertTrue(terms != null);
- assertTrue(terms.length == testTerms.length);
- for (int i = 0; i < terms.length; i++) {
- String term = terms[i].utf8ToString();
- //System.out.println("Term: " + term);
- assertTrue(term.equals(testTerms[i]));
- int[] positions = vector.getTermPositions(i);
- assertTrue(positions != null);
- assertTrue(positions.length == this.positions[i].length);
- for (int j = 0; j < positions.length; j++) {
- int position = positions[j];
- assertTrue(position == this.positions[i][j]);
- }
- TermVectorOffsetInfo[] offset = vector.getOffsets(i);
- assertTrue(offset != null);
- assertTrue(offset.length == this.offsets[i].length);
- for (int j = 0; j < offset.length; j++) {
- TermVectorOffsetInfo termVectorOffsetInfo = offset[j];
- assertTrue(termVectorOffsetInfo.equals(offsets[i][j]));
+ TermVectorsReader reader = Codec.getDefault().termVectorsFormat().vectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ Terms vector = reader.get(0).terms(testFields[0]);
+ assertNotNull(vector);
+ TermsEnum termsEnum = vector.iterator(null);
+ assertNotNull(termsEnum);
+ assertEquals(testTerms.length, vector.getUniqueTermCount());
+ DocsAndPositionsEnum dpEnum = null;
+ for (int i = 0; i < testTerms.length; i++) {
+ final BytesRef text = termsEnum.next();
+ assertNotNull(text);
+ String term = text.utf8ToString();
+ assertEquals(testTerms[i], term);
+
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ assertNotNull(dpEnum);
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ assertEquals(dpEnum.freq(), positions[i].length);
+ for (int j = 0; j < positions[i].length; j++) {
+ assertEquals(positions[i][j], dpEnum.nextPosition());
+ }
+ assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
+
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
+ final OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
+ assertNotNull(offsetAtt);
+ assertNotNull(dpEnum);
+ assertEquals(dpEnum.freq(), positions[i].length);
+ for (int j = 0; j < positions[i].length; j++) {
+ assertEquals(positions[i][j], dpEnum.nextPosition());
+ assertEquals(j*10, offsetAtt.startOffset());
+ assertEquals(j*10 + testTerms[i].length(), offsetAtt.endOffset());
}
+ assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
}
reader.close();
}
- public void testMapper() throws IOException {
- TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
- SortedTermVectorMapper mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
- reader.get(0, mapper);
- SortedSet<TermVectorEntry> set = mapper.getTermVectorEntrySet();
- assertTrue("set is null and it shouldn't be", set != null);
- //three fields, 4 terms, all terms are the same
- assertTrue("set Size: " + set.size() + " is not: " + 4, set.size() == 4);
- //Check offsets and positions
- for (Iterator<TermVectorEntry> iterator = set.iterator(); iterator.hasNext();) {
- TermVectorEntry tve = iterator.next();
- assertTrue("tve is null and it shouldn't be", tve != null);
- assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);
- assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);
-
- }
-
- mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
- reader.get(1, mapper);
- set = mapper.getTermVectorEntrySet();
- assertTrue("set is null and it shouldn't be", set != null);
- //three fields, 4 terms, all terms are the same
- assertTrue("set Size: " + set.size() + " is not: " + 4, set.size() == 4);
- //Should have offsets and positions b/c we are munging all the fields together
- for (Iterator<TermVectorEntry> iterator = set.iterator(); iterator.hasNext();) {
- TermVectorEntry tve = iterator.next();
- assertTrue("tve is null and it shouldn't be", tve != null);
- assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);
- assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);
-
- }
-
-
- FieldSortedTermVectorMapper fsMapper = new FieldSortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
- reader.get(0, fsMapper);
- Map<String,SortedSet<TermVectorEntry>> map = fsMapper.getFieldToTerms();
- assertTrue("map Size: " + map.size() + " is not: " + testFields.length, map.size() == testFields.length);
- for (Map.Entry<String,SortedSet<TermVectorEntry>> entry : map.entrySet()) {
- SortedSet<TermVectorEntry> sortedSet = entry.getValue();
- assertTrue("sortedSet Size: " + sortedSet.size() + " is not: " + 4, sortedSet.size() == 4);
- for (final TermVectorEntry tve : sortedSet) {
- assertTrue("tve is null and it shouldn't be", tve != null);
- //Check offsets and positions.
- assertTrue("tve is null and it shouldn't be", tve != null);
- String field = tve.getField();
- if (field.equals(testFields[0])) {
- //should have offsets
-
- assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);
- assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);
- }
- else if (field.equals(testFields[1])) {
- //should not have offsets
-
- assertTrue("tve.getOffsets() is not null and it shouldn't be", tve.getOffsets() == null);
- assertTrue("tve.getPositions() is not null and it shouldn't be", tve.getPositions() == null);
- }
- }
- }
- //Try mapper that ignores offs and positions
- fsMapper = new FieldSortedTermVectorMapper(true, true, new TermVectorEntryFreqSortedComparator());
- reader.get(0, fsMapper);
- map = fsMapper.getFieldToTerms();
- assertTrue("map Size: " + map.size() + " is not: " + testFields.length, map.size() == testFields.length);
- for (final Map.Entry<String,SortedSet<TermVectorEntry>> entry : map.entrySet()) {
- SortedSet<TermVectorEntry> sortedSet = entry.getValue();
- assertTrue("sortedSet Size: " + sortedSet.size() + " is not: " + 4, sortedSet.size() == 4);
- for (final TermVectorEntry tve : sortedSet) {
- assertTrue("tve is null and it shouldn't be", tve != null);
- //Check offsets and positions.
- assertTrue("tve is null and it shouldn't be", tve != null);
- String field = tve.getField();
- if (field.equals(testFields[0])) {
- //should have offsets
-
- assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() == null);
- assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() == null);
- }
- else if (field.equals(testFields[1])) {
- //should not have offsets
-
- assertTrue("tve.getOffsets() is not null and it shouldn't be", tve.getOffsets() == null);
- assertTrue("tve.getPositions() is not null and it shouldn't be", tve.getPositions() == null);
- }
- }
- }
-
- // test setDocumentNumber()
- IndexReader ir = IndexReader.open(dir, true);
- DocNumAwareMapper docNumAwareMapper = new DocNumAwareMapper();
- assertEquals(-1, docNumAwareMapper.getDocumentNumber());
-
- ir.getTermFreqVector(0, docNumAwareMapper);
- assertEquals(0, docNumAwareMapper.getDocumentNumber());
- docNumAwareMapper.setDocumentNumber(-1);
-
- ir.getTermFreqVector(1, docNumAwareMapper);
- assertEquals(1, docNumAwareMapper.getDocumentNumber());
- docNumAwareMapper.setDocumentNumber(-1);
-
- ir.getTermFreqVector(0, "f1", docNumAwareMapper);
- assertEquals(0, docNumAwareMapper.getDocumentNumber());
- docNumAwareMapper.setDocumentNumber(-1);
-
- ir.getTermFreqVector(1, "f2", docNumAwareMapper);
- assertEquals(1, docNumAwareMapper.getDocumentNumber());
- docNumAwareMapper.setDocumentNumber(-1);
-
- ir.getTermFreqVector(0, "f1", docNumAwareMapper);
- assertEquals(0, docNumAwareMapper.getDocumentNumber());
-
- ir.close();
- reader.close();
- }
-
-
/**
* Make sure exceptions and bad params are handled appropriately
*/
public void testBadParams() throws IOException {
TermVectorsReader reader = null;
try {
- reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ reader = Codec.getDefault().termVectorsFormat().vectorsReader(dir, seg, fieldInfos, newIOContext(random));
//Bad document number, good field number
- reader.get(50, testFields[0]);
- fail();
- } catch (IOException e) {
- // expected exception
- } finally {
- reader.close();
- }
- try {
- reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
- //Bad document number, no field
reader.get(50);
fail();
- } catch (IOException e) {
+ } catch (IllegalArgumentException e) {
// expected exception
} finally {
reader.close();
}
- try {
- reader = new TermVectorsReader(dir, seg, fieldInfos, newIOContext(random));
- //good document number, bad field number
- TermFreqVector vector = reader.get(0, "f50");
- assertTrue(vector == null);
- reader.close();
- } catch (IOException e) {
- fail();
- } finally {
- reader.close();
- }
- }
-
-
- public static class DocNumAwareMapper extends TermVectorMapper {
-
- public DocNumAwareMapper() {
- }
-
- private int documentNumber = -1;
-
- @Override
- public void setExpectations(String field, int numTerms, boolean storeOffsets, boolean storePositions) {
- if (documentNumber == -1) {
- throw new RuntimeException("Documentnumber should be set at this point!");
- }
- }
-
- @Override
- public void map(BytesRef term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {
- if (documentNumber == -1) {
- throw new RuntimeException("Documentnumber should be set at this point!");
- }
- }
-
- public int getDocumentNumber() {
- return documentNumber;
- }
-
- @Override
- public void setDocumentNumber(int documentNumber) {
- this.documentNumber = documentNumber;
- }
+ reader = Codec.getDefault().termVectorsFormat().vectorsReader(dir, seg, fieldInfos, newIOContext(random));
+ //good document number, bad field
+ Terms vector = reader.get(0).terms("f50");
+ assertNull(vector);
+ reader.close();
}
}