You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/01/18 23:28:20 UTC
svn commit: r1233096 [9/13] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/modules/analysis/kuromoji/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/maven/modules/a...
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java Wed Jan 18 22:28:07 2012
@@ -27,7 +27,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
@@ -35,6 +34,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp;
@@ -260,17 +260,17 @@ public class TestDuelingCodecs extends L
assertEquals(info, term, rightTermsEnum.next());
assertTermStats(leftTermsEnum, rightTermsEnum);
if (deep) {
- assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
- rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
- assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
- rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
+ assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, false),
+ rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, false));
+ assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, false),
+ rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, false));
assertPositionsSkipping(leftTermsEnum.docFreq(),
- leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
- rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
+ leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, false),
+ rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, false));
assertPositionsSkipping(leftTermsEnum.docFreq(),
- leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
- rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
+ leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, false),
+ rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, false));
// with freqs:
assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs, true),
@@ -341,6 +341,8 @@ public class TestDuelingCodecs extends L
for (int i = 0; i < freq; i++) {
assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
+ assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset());
+ assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset());
if (leftDocs.hasPayload()) {
assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
}
@@ -500,8 +502,6 @@ public class TestDuelingCodecs extends L
assertEquals(info, leftField.binaryValue(), rightField.binaryValue());
assertEquals(info, leftField.stringValue(), rightField.stringValue());
assertEquals(info, leftField.numericValue(), rightField.numericValue());
- assertEquals(info, leftField.numeric(), rightField.numeric());
- assertEquals(info, leftField.numericDataType(), rightField.numericDataType());
// TODO: should we check the FT at all?
}
@@ -516,13 +516,24 @@ public class TestDuelingCodecs extends L
assertFields(leftFields, rightFields, rarely());
}
}
+
+ private static Set<String> getDVFields(IndexReader reader) {
+ Set<String> fields = new HashSet<String>();
+ for(FieldInfo fi : ReaderUtil.getMergedFieldInfos(reader)) {
+ if (fi.hasDocValues()) {
+ fields.add(fi.name);
+ }
+ }
+
+ return fields;
+ }
/**
* checks that docvalues across all fields are equivalent
*/
public void assertDocValues(IndexReader leftReader, IndexReader rightReader) throws Exception {
- Set<String> leftValues = new HashSet<String>(leftReader.getFieldNames(FieldOption.DOC_VALUES));
- Set<String> rightValues = new HashSet<String>(rightReader.getFieldNames(FieldOption.DOC_VALUES));
+ Set<String> leftValues = getDVFields(leftReader);
+ Set<String> rightValues = getDVFields(rightReader);
assertEquals(info, leftValues, rightValues);
for (String field : leftValues) {
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java Wed Jan 18 22:28:07 2012
@@ -133,22 +133,15 @@ public class TestFieldInfos extends Luce
}
try {
readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
- random.nextBoolean(), random.nextBoolean());
+ random.nextBoolean());
fail("instance should be read only");
} catch (IllegalStateException e) {
// expected
}
try {
readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
- random.nextBoolean(), random.nextBoolean(), random.nextBoolean());
- fail("instance should be read only");
- } catch (IllegalStateException e) {
- // expected
- }
- try {
- readOnly.addOrUpdate("bogus", random.nextBoolean(), random.nextBoolean(),
- random.nextBoolean(), random.nextBoolean(), random.nextBoolean(),
- random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null);
+ random.nextBoolean(),
+ random.nextBoolean(), random.nextBoolean() ? IndexOptions.DOCS_ONLY : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, null);
fail("instance should be read only");
} catch (IllegalStateException e) {
// expected
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Wed Jan 18 22:28:07 2012
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericField;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -80,24 +81,18 @@ public class TestFieldsReader extends Lu
assertTrue(field != null);
assertTrue(field.fieldType().storeTermVectors());
- assertTrue(field.fieldType().storeTermVectorOffsets());
- assertTrue(field.fieldType().storeTermVectorPositions());
assertFalse(field.fieldType().omitNorms());
assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
field = (Field) doc.getField(DocHelper.TEXT_FIELD_3_KEY);
assertTrue(field != null);
assertFalse(field.fieldType().storeTermVectors());
- assertFalse(field.fieldType().storeTermVectorOffsets());
- assertFalse(field.fieldType().storeTermVectorPositions());
assertTrue(field.fieldType().omitNorms());
assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
field = (Field) doc.getField(DocHelper.NO_TF_KEY);
assertTrue(field != null);
assertFalse(field.fieldType().storeTermVectors());
- assertFalse(field.fieldType().storeTermVectorOffsets());
- assertFalse(field.fieldType().storeTermVectorPositions());
assertFalse(field.fieldType().omitNorms());
assertTrue(field.fieldType().indexOptions() == IndexOptions.DOCS_ONLY);
@@ -243,40 +238,42 @@ public class TestFieldsReader extends Lu
final NumericField.DataType[] typeAnswers = new NumericField.DataType[numDocs];
for(int id=0;id<numDocs;id++) {
Document doc = new Document();
- NumericField nf = new NumericField("nf", NumericField.TYPE_STORED);
- doc.add(nf);
+ final NumericField nf;
final Number answer;
final NumericField.DataType typeAnswer;
if (random.nextBoolean()) {
// float/double
if (random.nextBoolean()) {
final float f = random.nextFloat();
- nf.setFloatValue(f);
answer = Float.valueOf(f);
+ nf = new NumericField("nf", answer, NumericField.getFieldType(NumericField.DataType.FLOAT, true));
typeAnswer = NumericField.DataType.FLOAT;
} else {
final double d = random.nextDouble();
- nf.setDoubleValue(d);
answer = Double.valueOf(d);
+ nf = new NumericField("nf", answer, NumericField.getFieldType(NumericField.DataType.DOUBLE, true));
typeAnswer = NumericField.DataType.DOUBLE;
}
} else {
// int/long
if (random.nextBoolean()) {
final int i = random.nextInt();
- nf.setIntValue(i);
answer = Integer.valueOf(i);
+ nf = new NumericField("nf", answer, NumericField.getFieldType(NumericField.DataType.INT, true));
typeAnswer = NumericField.DataType.INT;
} else {
final long l = random.nextLong();
- nf.setLongValue(l);
answer = Long.valueOf(l);
+ nf = new NumericField("nf", answer, NumericField.getFieldType(NumericField.DataType.LONG, true));
typeAnswer = NumericField.DataType.LONG;
}
}
+ doc.add(nf);
answers[id] = answer;
typeAnswers[id] = typeAnswer;
- doc.add(new NumericField("id", Integer.MAX_VALUE).setIntValue(id));
+ FieldType ft = new FieldType(NumericField.getFieldType(NumericField.DataType.INT, false));
+ ft.setNumericPrecisionStep(Integer.MAX_VALUE);
+ doc.add(new NumericField("id", id, ft));
w.addDocument(doc);
}
final IndexReader r = w.getReader();
@@ -289,10 +286,8 @@ public class TestFieldsReader extends Lu
for(int docID=0;docID<sub.numDocs();docID++) {
final Document doc = sub.document(docID);
final Field f = (Field) doc.getField("nf");
- assertTrue("got f=" + f, f instanceof NumericField);
- final NumericField nf = (NumericField) f;
- assertEquals(answers[ids[docID]], nf.numericValue());
- assertSame(typeAnswers[ids[docID]], nf.numericDataType());
+ assertTrue("got f=" + f, f instanceof StoredField);
+ assertEquals(answers[ids[docID]], f.numericValue());
}
}
r.close();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestFilterIndexReader.java Wed Jan 18 22:28:07 2012
@@ -18,19 +18,20 @@ package org.apache.lucene.index;
*/
-import org.apache.lucene.util.LuceneTestCase;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.HashSet;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Bits;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.util.HashSet;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ReaderUtil;
public class TestFilterIndexReader extends LuceneTestCase {
@@ -90,8 +91,8 @@ public class TestFilterIndexReader exten
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
- return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in));
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+ return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in, needsOffsets));
}
}
@@ -121,6 +122,11 @@ public class TestFilterIndexReader exten
public Fields fields() throws IOException {
return new TestFields(super.fields());
}
+
+ @Override
+ public FieldInfos getFieldInfos() {
+ return ReaderUtil.getMergedFieldInfos(in);
+ }
}
/**
@@ -129,6 +135,7 @@ public class TestFilterIndexReader exten
*/
public void testFilterIndexReader() throws Exception {
Directory directory = newDirectory();
+
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
Document d1 = new Document();
@@ -146,6 +153,10 @@ public class TestFilterIndexReader exten
writer.close();
Directory target = newDirectory();
+
+ // We mess with the postings so this can fail:
+ ((MockDirectoryWrapper) target).setCrossCheckTermVectorsOnClose(false);
+
writer = new IndexWriter(target, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
IndexReader reader = new TestReader(IndexReader.open(directory));
writer.addIndexes(reader);
@@ -161,7 +172,7 @@ public class TestFilterIndexReader exten
assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(new BytesRef("one")));
DocsAndPositionsEnum positions = terms.docsAndPositions(MultiFields.getLiveDocs(reader),
- null);
+ null, false);
while (positions.nextDoc() != DocsEnum.NO_MORE_DOCS) {
assertTrue((positions.docID() % 2) == 1);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Wed Jan 18 22:28:07 2012
@@ -30,13 +30,12 @@ import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
-import org.apache.lucene.document.BinaryField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
@@ -46,6 +45,7 @@ import org.apache.lucene.store.NoSuchDir
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
@@ -102,11 +102,11 @@ public class TestIndexReader extends Luc
writer.close();
// set up reader
IndexReader reader = IndexReader.open(d);
- Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL);
- assertTrue(fieldNames.contains("keyword"));
- assertTrue(fieldNames.contains("text"));
- assertTrue(fieldNames.contains("unindexed"));
- assertTrue(fieldNames.contains("unstored"));
+ FieldInfos fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+ assertNotNull(fieldInfos.fieldInfo("keyword"));
+ assertNotNull(fieldInfos.fieldInfo("text"));
+ assertNotNull(fieldInfos.fieldInfo("unindexed"));
+ assertNotNull(fieldInfos.fieldInfo("unstored"));
reader.close();
// add more documents
writer = new IndexWriter(
@@ -160,61 +160,66 @@ public class TestIndexReader extends Luc
}
writer.close();
+
// verify fields again
reader = IndexReader.open(d);
- fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL);
- assertEquals(13, fieldNames.size()); // the following fields
- assertTrue(fieldNames.contains("keyword"));
- assertTrue(fieldNames.contains("text"));
- assertTrue(fieldNames.contains("unindexed"));
- assertTrue(fieldNames.contains("unstored"));
- assertTrue(fieldNames.contains("keyword2"));
- assertTrue(fieldNames.contains("text2"));
- assertTrue(fieldNames.contains("unindexed2"));
- assertTrue(fieldNames.contains("unstored2"));
- assertTrue(fieldNames.contains("tvnot"));
- assertTrue(fieldNames.contains("termvector"));
- assertTrue(fieldNames.contains("tvposition"));
- assertTrue(fieldNames.contains("tvoffset"));
- assertTrue(fieldNames.contains("tvpositionoffset"));
+ fieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+
+ Collection<String> allFieldNames = new HashSet<String>();
+ Collection<String> indexedFieldNames = new HashSet<String>();
+ Collection<String> notIndexedFieldNames = new HashSet<String>();
+ Collection<String> tvFieldNames = new HashSet<String>();
+
+ for(FieldInfo fieldInfo : fieldInfos) {
+ final String name = fieldInfo.name;
+ allFieldNames.add(name);
+ if (fieldInfo.isIndexed) {
+ indexedFieldNames.add(name);
+ } else {
+ notIndexedFieldNames.add(name);
+ }
+ if (fieldInfo.storeTermVector) {
+ tvFieldNames.add(name);
+ }
+ }
+
+ assertTrue(allFieldNames.contains("keyword"));
+ assertTrue(allFieldNames.contains("text"));
+ assertTrue(allFieldNames.contains("unindexed"));
+ assertTrue(allFieldNames.contains("unstored"));
+ assertTrue(allFieldNames.contains("keyword2"));
+ assertTrue(allFieldNames.contains("text2"));
+ assertTrue(allFieldNames.contains("unindexed2"));
+ assertTrue(allFieldNames.contains("unstored2"));
+ assertTrue(allFieldNames.contains("tvnot"));
+ assertTrue(allFieldNames.contains("termvector"));
+ assertTrue(allFieldNames.contains("tvposition"));
+ assertTrue(allFieldNames.contains("tvoffset"));
+ assertTrue(allFieldNames.contains("tvpositionoffset"));
// verify that only indexed fields were returned
- fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
- assertEquals(11, fieldNames.size()); // 6 original + the 5 termvector fields
- assertTrue(fieldNames.contains("keyword"));
- assertTrue(fieldNames.contains("text"));
- assertTrue(fieldNames.contains("unstored"));
- assertTrue(fieldNames.contains("keyword2"));
- assertTrue(fieldNames.contains("text2"));
- assertTrue(fieldNames.contains("unstored2"));
- assertTrue(fieldNames.contains("tvnot"));
- assertTrue(fieldNames.contains("termvector"));
- assertTrue(fieldNames.contains("tvposition"));
- assertTrue(fieldNames.contains("tvoffset"));
- assertTrue(fieldNames.contains("tvpositionoffset"));
+ assertEquals(11, indexedFieldNames.size()); // 6 original + the 5 termvector fields
+ assertTrue(indexedFieldNames.contains("keyword"));
+ assertTrue(indexedFieldNames.contains("text"));
+ assertTrue(indexedFieldNames.contains("unstored"));
+ assertTrue(indexedFieldNames.contains("keyword2"));
+ assertTrue(indexedFieldNames.contains("text2"));
+ assertTrue(indexedFieldNames.contains("unstored2"));
+ assertTrue(indexedFieldNames.contains("tvnot"));
+ assertTrue(indexedFieldNames.contains("termvector"));
+ assertTrue(indexedFieldNames.contains("tvposition"));
+ assertTrue(indexedFieldNames.contains("tvoffset"));
+ assertTrue(indexedFieldNames.contains("tvpositionoffset"));
// verify that only unindexed fields were returned
- fieldNames = reader.getFieldNames(IndexReader.FieldOption.UNINDEXED);
- assertEquals(2, fieldNames.size()); // the following fields
- assertTrue(fieldNames.contains("unindexed"));
- assertTrue(fieldNames.contains("unindexed2"));
+ assertEquals(2, notIndexedFieldNames.size()); // the following fields
+ assertTrue(notIndexedFieldNames.contains("unindexed"));
+ assertTrue(notIndexedFieldNames.contains("unindexed2"));
// verify index term vector fields
- fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR);
- assertEquals(1, fieldNames.size()); // 1 field has term vector only
- assertTrue(fieldNames.contains("termvector"));
-
- fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION);
- assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
- assertTrue(fieldNames.contains("tvposition"));
-
- fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET);
- assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
- assertTrue(fieldNames.contains("tvoffset"));
-
- fieldNames = reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET);
- assertEquals(1, fieldNames.size()); // 4 fields are indexed with term vectors
- assertTrue(fieldNames.contains("tvpositionoffset"));
+ assertEquals(tvFieldNames.toString(), 4, tvFieldNames.size()); // 4 field has term vector only
+ assertTrue(tvFieldNames.contains("termvector"));
+
reader.close();
d.close();
}
@@ -292,7 +297,7 @@ public class TestIndexReader extends Luc
writer.close();
writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
Document doc = new Document();
- doc.add(new BinaryField("bin1", bin));
+ doc.add(new StoredField("bin1", bin));
doc.add(new TextField("junk", "junk text"));
writer.addDocument(doc);
writer.close();
@@ -519,19 +524,19 @@ public class TestIndexReader extends Luc
}
// check field names
- Collection<String> fields1 = index1.getFieldNames(FieldOption.ALL);
- Collection<String> fields2 = index1.getFieldNames(FieldOption.ALL);
- assertEquals("IndexReaders have different numbers of fields.", fields1.size(), fields2.size());
- Iterator<String> it1 = fields1.iterator();
- Iterator<String> it2 = fields1.iterator();
- while (it1.hasNext()) {
- assertEquals("Different field names.", it1.next(), it2.next());
+ FieldInfos fieldInfos1 = ReaderUtil.getMergedFieldInfos(index1);
+ FieldInfos fieldInfos2 = ReaderUtil.getMergedFieldInfos(index2);
+ assertEquals("IndexReaders have different numbers of fields.", fieldInfos1.size(), fieldInfos2.size());
+ final int numFields = fieldInfos1.size();
+ for(int fieldID=0;fieldID<numFields;fieldID++) {
+ final FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldID);
+ final FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldID);
+ assertEquals("Different field names.", fieldInfo1.name, fieldInfo2.name);
}
// check norms
- it1 = fields1.iterator();
- while (it1.hasNext()) {
- String curField = it1.next();
+ for(FieldInfo fieldInfo : fieldInfos1) {
+ String curField = fieldInfo.name;
DocValues norms1 = MultiDocValues.getNormDocValues(index1, curField);
DocValues norms2 = MultiDocValues.getNormDocValues(index2, curField);
if (norms1 != null && norms2 != null)
@@ -598,8 +603,8 @@ public class TestIndexReader extends Luc
while(enum1.next() != null) {
assertEquals("Different terms", enum1.term(), enum2.next());
- DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null);
- DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null);
+ DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null, false);
+ DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null, false);
while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Jan 18 22:28:07 2012
@@ -21,15 +21,10 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
-import java.util.Random;
import java.util.Set;
import org.apache.lucene.analysis.*;
@@ -37,13 +32,14 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
-import org.apache.lucene.document.BinaryField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
@@ -51,7 +47,6 @@ import org.apache.lucene.search.PhraseQu
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
@@ -905,7 +900,8 @@ public class TestIndexWriter extends Luc
DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(),
MultiFields.getLiveDocs(s.getIndexReader()),
"field",
- new BytesRef("a"));
+ new BytesRef("a"),
+ false);
assertTrue(tps.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(1, tps.freq());
@@ -927,7 +923,7 @@ public class TestIndexWriter extends Luc
b[i] = (byte) (i+77);
Document doc = new Document();
- Field f = new BinaryField("binary", b, 10, 17);
+ Field f = new StoredField("binary", b, 10, 17);
byte[] bx = f.binaryValue().bytes;
assertTrue(bx != null);
assertEquals(50, bx.length);
@@ -970,14 +966,14 @@ public class TestIndexWriter extends Luc
Terms tpv = r.getTermVectors(0).terms("field");
TermsEnum termsEnum = tpv.iterator(null);
assertNotNull(termsEnum.next());
- DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+ DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
assertNotNull(dpEnum);
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertEquals(1, dpEnum.freq());
assertEquals(100, dpEnum.nextPosition());
assertNotNull(termsEnum.next());
- dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
assertNotNull(dpEnum);
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertEquals(1, dpEnum.freq());
@@ -1183,11 +1179,11 @@ public class TestIndexWriter extends Luc
Document doc = new Document();
- FieldType customType = new FieldType(BinaryField.TYPE_STORED);
+ FieldType customType = new FieldType(StoredField.TYPE);
customType.setTokenized(true);
- customType.setIndexed(true);
Field f = new Field("binary", b, 10, 17, customType);
+ customType.setIndexed(true);
f.setTokenStream(new MockTokenizer(new StringReader("doc1field1"), MockTokenizer.WHITESPACE, false));
FieldType customType2 = new FieldType(TextField.TYPE_STORED);
@@ -1640,7 +1636,7 @@ public class TestIndexWriter extends Luc
// Make sure position is still incremented when
// massive term is skipped:
- DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another"));
+ DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another"), false);
assertEquals(0, tps.nextDoc());
assertEquals(1, tps.freq());
assertEquals(3, tps.nextPosition());
@@ -1679,7 +1675,7 @@ public class TestIndexWriter extends Luc
w.close();
assertEquals(1, reader.docFreq(new Term("content", bigTerm)));
- FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(reader, "content", random.nextBoolean());
+ FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(new SlowMultiReaderWrapper(reader), "content", random.nextBoolean());
assertEquals(5, dti.numOrd()); // +1 for null ord
assertEquals(4, dti.size());
assertEquals(bigTermBytesRef, dti.lookup(3, new BytesRef()));
@@ -1767,4 +1763,27 @@ public class TestIndexWriter extends Luc
w1.close();
d.close();
}
+
+ public void testChangeIndexOptions() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir,
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+
+ FieldType docsAndFreqs = new FieldType(TextField.TYPE_UNSTORED);
+ docsAndFreqs.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+
+ FieldType docsOnly = new FieldType(TextField.TYPE_UNSTORED);
+ docsOnly.setIndexOptions(IndexOptions.DOCS_ONLY);
+
+ Document doc = new Document();
+ doc.add(new Field("field", "a b c", docsAndFreqs));
+ w.addDocument(doc);
+ w.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new Field("field", "a b c", docsOnly));
+ w.addDocument(doc);
+ w.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Wed Jan 18 22:28:07 2012
@@ -18,10 +18,12 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
@@ -29,19 +31,18 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
-import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.util._TestUtil;
public class TestIndexWriterReader extends LuceneTestCase {
@@ -780,7 +781,8 @@ public class TestIndexWriterReader exten
assertEquals(0, excs.size());
r.close();
- assertEquals(0, dir1.getOpenDeletedFiles().size());
+ final Collection<String> openDeletedFiles = dir1.getOpenDeletedFiles();
+ assertEquals("openDeleted=" + openDeletedFiles, 0, openDeletedFiles.size());
writer.close();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexableField.java Wed Jan 18 22:28:07 2012
@@ -25,14 +25,10 @@ import java.util.Iterator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.NumericField.DataType;
-import org.apache.lucene.document.NumericField;
import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
@@ -85,6 +81,11 @@ public class TestIndexableField extends
public FieldInfo.IndexOptions indexOptions() {
return FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
}
+
+ @Override
+ public DocValues.Type docValueType() {
+ return null;
+ }
};
public MyField(int counter) {
@@ -117,7 +118,7 @@ public class TestIndexableField extends
@Override
public String stringValue() {
final int fieldID = counter%10;
- if (fieldID != 3 && fieldID != 7 && fieldID != 9) {
+ if (fieldID != 3 && fieldID != 7) {
return "text " + counter;
} else {
return null;
@@ -133,20 +134,9 @@ public class TestIndexableField extends
}
}
- // Numeric field:
- @Override
- public boolean numeric() {
- return counter%10 == 9;
- }
-
- @Override
- public DataType numericDataType() {
- return DataType.INT;
- }
-
@Override
public Number numericValue() {
- return counter;
+ return null;
}
@Override
@@ -154,24 +144,10 @@ public class TestIndexableField extends
return fieldType;
}
- // TODO: randomly enable doc values
- @Override
- public DocValue docValue() {
- return null;
- }
-
- @Override
- public DocValues.Type docValueType() {
- return null;
- }
-
@Override
public TokenStream tokenStream(Analyzer analyzer) throws IOException {
- if (numeric()) {
- return new NumericField(name()).setIntValue(counter).tokenStream(analyzer);
- }
return readerValue() != null ? analyzer.tokenStream(name(), readerValue()) :
- analyzer.tokenStream(name(), new StringReader(stringValue()));
+ analyzer.tokenStream(name(), new StringReader(stringValue()));
}
}
@@ -253,7 +229,6 @@ public class TestIndexableField extends
final boolean stored = (counter&1) == 0 || fieldID == 3;
final boolean binary = fieldID == 3;
final boolean indexed = fieldID != 3;
- final boolean numeric = fieldID == 9;
final String stringValue;
if (fieldID != 3 && fieldID != 9) {
@@ -274,11 +249,6 @@ public class TestIndexableField extends
for(int idx=0;idx<10;idx++) {
assertEquals((byte) (idx+counter), b.bytes[b.offset+idx]);
}
- } else if (numeric) {
- assertTrue(f instanceof NumericField);
- final NumericField nf = (NumericField) f;
- assertEquals(NumericField.DataType.INT, nf.numericDataType());
- assertEquals(counter, nf.numericValue().intValue());
} else {
assert stringValue != null;
assertEquals(stringValue, f.stringValue());
@@ -293,14 +263,14 @@ public class TestIndexableField extends
TermsEnum termsEnum = tfv.iterator(null);
assertEquals(new BytesRef(""+counter), termsEnum.next());
assertEquals(1, termsEnum.totalTermFreq());
- DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null);
+ DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertEquals(1, dpEnum.freq());
assertEquals(1, dpEnum.nextPosition());
assertEquals(new BytesRef("text"), termsEnum.next());
assertEquals(1, termsEnum.totalTermFreq());
- dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertEquals(1, dpEnum.freq());
assertEquals(0, dpEnum.nextPosition());
@@ -314,26 +284,19 @@ public class TestIndexableField extends
assertTrue(vectors == null || vectors.terms(name) == null);
}
- if (numeric) {
- NumericRangeQuery nrq = NumericRangeQuery.newIntRange(name, counter, counter, true, true);
- final TopDocs hits2 = s.search(nrq, 1);
- assertEquals(1, hits2.totalHits);
- assertEquals(docID, hits2.scoreDocs[0].doc);
- } else {
- BooleanQuery bq = new BooleanQuery();
- bq.add(new TermQuery(new Term("id", ""+id)), BooleanClause.Occur.MUST);
- bq.add(new TermQuery(new Term(name, "text")), BooleanClause.Occur.MUST);
- final TopDocs hits2 = s.search(bq, 1);
- assertEquals(1, hits2.totalHits);
- assertEquals(docID, hits2.scoreDocs[0].doc);
-
- bq = new BooleanQuery();
- bq.add(new TermQuery(new Term("id", ""+id)), BooleanClause.Occur.MUST);
- bq.add(new TermQuery(new Term(name, ""+counter)), BooleanClause.Occur.MUST);
- final TopDocs hits3 = s.search(bq, 1);
- assertEquals(1, hits3.totalHits);
- assertEquals(docID, hits3.scoreDocs[0].doc);
- }
+ BooleanQuery bq = new BooleanQuery();
+ bq.add(new TermQuery(new Term("id", ""+id)), BooleanClause.Occur.MUST);
+ bq.add(new TermQuery(new Term(name, "text")), BooleanClause.Occur.MUST);
+ final TopDocs hits2 = s.search(bq, 1);
+ assertEquals(1, hits2.totalHits);
+ assertEquals(docID, hits2.scoreDocs[0].doc);
+
+ bq = new BooleanQuery();
+ bq.add(new TermQuery(new Term("id", ""+id)), BooleanClause.Occur.MUST);
+ bq.add(new TermQuery(new Term(name, ""+counter)), BooleanClause.Occur.MUST);
+ final TopDocs hits3 = s.search(bq, 1);
+ assertEquals(1, hits3.totalHits);
+ assertEquals(docID, hits3.scoreDocs[0].doc);
}
counter++;
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Wed Jan 18 22:28:07 2012
@@ -156,7 +156,8 @@ public class TestLazyProxSkipping extend
DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
this.field,
- new BytesRef("b"));
+ new BytesRef("b"),
+ false);
for (int i = 0; i < 10; i++) {
tp.nextDoc();
@@ -167,7 +168,8 @@ public class TestLazyProxSkipping extend
tp = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
this.field,
- new BytesRef("a"));
+ new BytesRef("a"),
+ false);
for (int i = 0; i < 10; i++) {
tp.nextDoc();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Wed Jan 18 22:28:07 2012
@@ -173,7 +173,7 @@ public class TestLongPostings extends Lu
System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1);
}
- final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term));
+ final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term), false);
int docID = -1;
while(docID < DocsEnum.NO_MORE_DOCS) {
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Wed Jan 18 22:28:07 2012
@@ -117,8 +117,8 @@ public class TestMaxTermFrequency extend
}
@Override
- public byte computeNorm(FieldInvertState state) {
- return encodeNormValue((float) state.getMaxTermFrequency());
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte(encodeNormValue((float) state.getMaxTermFrequency()));
}
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Wed Jan 18 22:28:07 2012
@@ -86,7 +86,8 @@ public class TestMultiLevelSkipList exte
counter = 0;
DocsAndPositionsEnum tp = reader.termPositionsEnum(reader.getLiveDocs(),
term.field(),
- new BytesRef(term.text()));
+ new BytesRef(term.text()),
+ false);
checkSkipTo(tp, 14, 185); // no skips
checkSkipTo(tp, 17, 190); // one skip on level 0
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java Wed Jan 18 22:28:07 2012
@@ -18,20 +18,20 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Random;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -39,7 +39,8 @@ import org.apache.lucene.util.LuceneTest
* separate norms, addDocument, addIndexes, forceMerge.
*/
public class TestNorms extends LuceneTestCase {
-
+ final String byteTestField = "normsTestByte";
+
class CustomNormEncodingSimilarity extends DefaultSimilarity {
@Override
public byte encodeNormValue(float f) {
@@ -52,8 +53,8 @@ public class TestNorms extends LuceneTes
}
@Override
- public byte computeNorm(FieldInvertState state) {
- return encodeNormValue((float) state.getLength());
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setByte(encodeNormValue((float) state.getLength()));
}
}
@@ -93,4 +94,160 @@ public class TestNorms extends LuceneTes
reader.close();
dir.close();
}
+
+ public void testMaxByteNorms() throws IOException {
+ Directory dir = newDirectory();
+ buildIndex(dir, true);
+ IndexReader open = new SlowMultiReaderWrapper(IndexReader.open(dir));
+ DocValues normValues = open.normValues(byteTestField);
+ assertNotNull(normValues);
+ Source source = normValues.getSource();
+ assertTrue(source.hasArray());
+ assertEquals(Type.FIXED_INTS_8, normValues.type());
+ byte[] norms = (byte[]) source.getArray();
+ for (int i = 0; i < open.maxDoc(); i++) {
+ Document document = open.document(i);
+ int expected = Integer.parseInt(document.get(byteTestField));
+ assertEquals((byte)expected, norms[i]);
+ }
+ open.close();
+ dir.close();
+ }
+
+ /**
+ * this test randomly creates segments with or without norms but not omitting
+ * norms. The similarity used doesn't write a norm value if writeNorms = false is
+ * passed. This differs from omitNorm since norms are simply not written for this segment
+ * while merging fills in default values based on the Norm {@link Type}
+ */
+ public void testNormsNotPresent() throws IOException {
+ Directory dir = newDirectory();
+ boolean firstWriteNorm = random.nextBoolean();
+ buildIndex(dir, firstWriteNorm);
+
+ Directory otherDir = newDirectory();
+ boolean secondWriteNorm = random.nextBoolean();
+ buildIndex(otherDir, secondWriteNorm);
+
+ IndexReader reader = new SlowMultiReaderWrapper(IndexReader.open(otherDir));
+ FieldInfos fieldInfos = reader.getFieldInfos();
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(byteTestField);
+ assertFalse(fieldInfo.omitNorms);
+ assertTrue(fieldInfo.isIndexed);
+ if (secondWriteNorm) {
+ assertTrue(fieldInfo.normsPresent());
+ } else {
+ assertFalse(fieldInfo.normsPresent());
+ }
+
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
+ writer.addIndexes(reader);
+ IndexReader mergedReader = new SlowMultiReaderWrapper(writer.getReader());
+ if (!firstWriteNorm && !secondWriteNorm) {
+ DocValues normValues = mergedReader.normValues(byteTestField);
+ assertNull(normValues);
+ FieldInfo fi = mergedReader.getFieldInfos().fieldInfo(byteTestField);
+ assertFalse(fi.omitNorms);
+ assertTrue(fi.isIndexed);
+ assertFalse(fi.normsPresent());
+ } else {
+ FieldInfo fi = mergedReader.getFieldInfos().fieldInfo(byteTestField);
+ assertFalse(fi.omitNorms);
+ assertTrue(fi.isIndexed);
+ assertTrue(fi.normsPresent());
+
+ DocValues normValues = mergedReader.normValues(byteTestField);
+ assertNotNull(normValues);
+ Source source = normValues.getSource();
+ assertTrue(source.hasArray());
+ assertEquals(Type.FIXED_INTS_8, normValues.type());
+ byte[] norms = (byte[]) source.getArray();
+ for (int i = 0; i < mergedReader.maxDoc(); i++) {
+ Document document = mergedReader.document(i);
+ int expected = Integer.parseInt(document.get(byteTestField));
+ assertEquals((byte) expected, norms[i]);
+ }
+ }
+ mergedReader.close();
+ reader.close();
+
+ writer.close();
+ dir.close();
+ otherDir.close();
+ }
+
+ public void buildIndex(Directory dir, boolean writeNorms) throws IOException,
+ CorruptIndexException {
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ SimilarityProvider provider = new MySimProvider(writeNorms);
+ config.setSimilarityProvider(provider);
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir, config);
+ final LineFileDocs docs = new LineFileDocs(random);
+ int num = atLeast(100);
+ for (int i = 0; i < num; i++) {
+ Document doc = docs.nextDoc();
+ int boost = writeNorms ? 1 + random.nextInt(255) : 0;
+ Field f = new Field(byteTestField, "" + boost,
+ TextField.TYPE_STORED);
+ f.setBoost(boost);
+ doc.add(f);
+ writer.addDocument(doc);
+ doc.removeField(byteTestField);
+ if (rarely()) {
+ writer.commit();
+ }
+ }
+ writer.commit();
+ writer.close();
+ }
+
+
+ public class MySimProvider implements SimilarityProvider {
+ SimilarityProvider delegate = new DefaultSimilarityProvider();
+ private boolean writeNorms;
+ public MySimProvider(boolean writeNorms) {
+ this.writeNorms = writeNorms;
+ }
+ @Override
+ public float queryNorm(float sumOfSquaredWeights) {
+
+ return delegate.queryNorm(sumOfSquaredWeights);
+ }
+
+ @Override
+ public Similarity get(String field) {
+ if (byteTestField.equals(field)) {
+ return new ByteEncodingBoostSimilarity(writeNorms);
+ } else {
+ return delegate.get(field);
+ }
+ }
+
+ @Override
+ public float coord(int overlap, int maxOverlap) {
+ return delegate.coord(overlap, maxOverlap);
+ }
+ }
+
+
+ public static class ByteEncodingBoostSimilarity extends DefaultSimilarity {
+
+ private boolean writeNorms;
+
+ public ByteEncodingBoostSimilarity(boolean writeNorms) {
+ this.writeNorms = writeNorms;
+ }
+
+ @Override
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ if (writeNorms) {
+ int boost = (int) state.getBoost();
+ norm.setByte((byte) (0xFF & boost));
+ }
+ }
+ }
+
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java Wed Jan 18 22:28:07 2012
@@ -67,7 +67,7 @@ public class TestOmitNorms extends Lucen
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
- FieldInfos fi = reader.fieldInfos();
+ FieldInfos fi = reader.getFieldInfos();
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms);
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);
@@ -121,7 +121,7 @@ public class TestOmitNorms extends Lucen
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
- FieldInfos fi = reader.fieldInfos();
+ FieldInfos fi = reader.getFieldInfos();
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f1").omitNorms);
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);
@@ -169,7 +169,7 @@ public class TestOmitNorms extends Lucen
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
- FieldInfos fi = reader.fieldInfos();
+ FieldInfos fi = reader.getFieldInfos();
assertTrue("OmitNorms field bit should not be set.", !fi.fieldInfo("f1").omitNorms);
assertTrue("OmitNorms field bit should be set.", fi.fieldInfo("f2").omitNorms);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java Wed Jan 18 22:28:07 2012
@@ -51,7 +51,7 @@ public class TestOmitPositions extends L
IndexReader reader = w.getReader();
w.close();
- assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test")));
+ assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test"), false));
DocsEnum de = _TestUtil.docs(random, reader, "foo", new BytesRef("test"), null, null, true);
while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -154,7 +154,7 @@ public class TestOmitPositions extends L
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
- FieldInfos fi = reader.fieldInfos();
+ FieldInfos fi = reader.getFieldInfos();
// docs + docs = docs
assertEquals(IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
// docs + docs/freqs = docs
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Wed Jan 18 22:28:07 2012
@@ -44,8 +44,7 @@ public class TestOmitTf extends LuceneTe
public float coord(int overlap, int maxOverlap) { return 1.0f; }
public Similarity get(String field) {
return new TFIDFSimilarity() {
-
- @Override public byte computeNorm(FieldInvertState state) { return encodeNormValue(state.getBoost()); }
+ @Override public void computeNorm(FieldInvertState state, Norm norm) { norm.setByte(encodeNormValue(state.getBoost())); }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(int docFreq, int numDocs) { return 1.0f; }
@@ -101,7 +100,7 @@ public class TestOmitTf extends LuceneTe
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
- FieldInfos fi = reader.fieldInfos();
+ FieldInfos fi = reader.getFieldInfos();
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
@@ -153,7 +152,7 @@ public class TestOmitTf extends LuceneTe
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
- FieldInfos fi = reader.fieldInfos();
+ FieldInfos fi = reader.getFieldInfos();
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f1").indexOptions);
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
@@ -196,7 +195,7 @@ public class TestOmitTf extends LuceneTe
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
- FieldInfos fi = reader.fieldInfos();
+ FieldInfos fi = reader.getFieldInfos();
assertEquals("OmitTermFreqAndPositions field bit should not be set.", IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.fieldInfo("f1").indexOptions);
assertEquals("OmitTermFreqAndPositions field bit should be set.", IndexOptions.DOCS_ONLY, fi.fieldInfo("f2").indexOptions);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestParallelReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestParallelReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestParallelReader.java Wed Jan 18 22:28:07 2012
@@ -18,14 +18,12 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Collection;
import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -75,12 +73,12 @@ public class TestParallelReader extends
ParallelReader pr = new ParallelReader();
pr.add(IndexReader.open(dir1));
pr.add(IndexReader.open(dir2));
- Collection<String> fieldNames = pr.getFieldNames(IndexReader.FieldOption.ALL);
- assertEquals(4, fieldNames.size());
- assertTrue(fieldNames.contains("f1"));
- assertTrue(fieldNames.contains("f2"));
- assertTrue(fieldNames.contains("f3"));
- assertTrue(fieldNames.contains("f4"));
+ FieldInfos fieldInfos = pr.getFieldInfos();
+ assertEquals(4, fieldInfos.size());
+ assertNotNull(fieldInfos.fieldInfo("f1"));
+ assertNotNull(fieldInfos.fieldInfo("f2"));
+ assertNotNull(fieldInfos.fieldInfo("f3"));
+ assertNotNull(fieldInfos.fieldInfo("f4"));
pr.close();
dir1.close();
dir2.close();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java Wed Jan 18 22:28:07 2012
@@ -155,7 +155,7 @@ public class TestPayloadProcessorProvide
IndexReader reader = IndexReader.open(dir);
try {
int numPayloads = 0;
- DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text);
+ DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text, false);
while (tpe.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
tpe.nextPosition();
if (tpe.hasPayload()) {
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestPayloads.java Wed Jan 18 22:28:07 2012
@@ -112,7 +112,7 @@ public class TestPayloads extends Lucene
writer.close();
SegmentReader reader = getOnlySegmentReader(IndexReader.open(ram));
- FieldInfos fi = reader.fieldInfos();
+ FieldInfos fi = reader.getFieldInfos();
assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads);
assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads);
assertFalse("Payload field bit should not be set.", fi.fieldInfo("f3").storePayloads);
@@ -139,7 +139,7 @@ public class TestPayloads extends Lucene
writer.close();
reader = getOnlySegmentReader(IndexReader.open(ram));
- fi = reader.fieldInfos();
+ fi = reader.getFieldInfos();
assertFalse("Payload field bit should not be set.", fi.fieldInfo("f1").storePayloads);
assertTrue("Payload field bit should be set.", fi.fieldInfo("f2").storePayloads);
assertTrue("Payload field bit should be set.", fi.fieldInfo("f3").storePayloads);
@@ -222,7 +222,8 @@ public class TestPayloads extends Lucene
tps[i] = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
terms[i].field(),
- new BytesRef(terms[i].text()));
+ new BytesRef(terms[i].text()),
+ false);
}
while (tps[0].nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -259,7 +260,8 @@ public class TestPayloads extends Lucene
DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
terms[0].field(),
- new BytesRef(terms[0].text()));
+ new BytesRef(terms[0].text()),
+ false);
tp.nextDoc();
tp.nextPosition();
// NOTE: prior rev of this test was failing to first
@@ -287,7 +289,8 @@ public class TestPayloads extends Lucene
tp = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
terms[1].field(),
- new BytesRef(terms[1].text()));
+ new BytesRef(terms[1].text()),
+ false);
tp.nextDoc();
tp.nextPosition();
assertEquals("Wrong payload length.", 1, tp.getPayload().length);
@@ -330,7 +333,8 @@ public class TestPayloads extends Lucene
tp = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
fieldName,
- new BytesRef(singleTerm));
+ new BytesRef(singleTerm),
+ false);
tp.nextDoc();
tp.nextPosition();
@@ -532,7 +536,7 @@ public class TestPayloads extends Lucene
DocsAndPositionsEnum tp = null;
while (terms.next() != null) {
String termText = terms.term().utf8ToString();
- tp = terms.docsAndPositions(liveDocs, tp);
+ tp = terms.docsAndPositions(liveDocs, tp, false);
while(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
int freq = tp.freq();
for (int i = 0; i < freq; i++) {
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Wed Jan 18 22:28:07 2012
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Collection;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
@@ -30,6 +29,7 @@ import org.apache.lucene.util.InfoStream
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
+
public class TestSegmentMerger extends LuceneTestCase {
//The variables for the new merged segment
private Directory mergedDir;
@@ -107,10 +107,15 @@ public class TestSegmentMerger extends L
assertTrue(termDocs != null);
assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS);
- Collection<String> stored = mergedReader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR);
- assertTrue(stored != null);
+ int tvCount = 0;
+ for(FieldInfo fieldInfo : mergedReader.getFieldInfos()) {
+ if (fieldInfo.storeTermVector) {
+ tvCount++;
+ }
+ }
+
//System.out.println("stored size: " + stored.size());
- assertTrue("We do not have 3 fields that were indexed with term vector",stored.size() == 3);
+ assertEquals("We do not have 3 fields that were indexed with term vector", 3, tvCount);
Terms vector = mergedReader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
assertNotNull(vector);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java Wed Jan 18 22:28:07 2012
@@ -19,7 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collection;
-import java.util.Iterator;
+import java.util.HashSet;
import java.util.List;
import org.apache.lucene.document.Document;
@@ -74,33 +74,42 @@ public class TestSegmentReader extends L
}
public void testGetFieldNameVariations() {
- Collection<String> result = reader.getFieldNames(IndexReader.FieldOption.ALL);
- assertTrue(result != null);
- assertTrue(result.size() == DocHelper.all.size());
- for (Iterator<String> iter = result.iterator(); iter.hasNext();) {
- String s = iter.next();
- //System.out.println("Name: " + s);
+ Collection<String> allFieldNames = new HashSet<String>();
+ Collection<String> indexedFieldNames = new HashSet<String>();
+ Collection<String> notIndexedFieldNames = new HashSet<String>();
+ Collection<String> tvFieldNames = new HashSet<String>();
+ Collection<String> noTVFieldNames = new HashSet<String>();
+
+ for(FieldInfo fieldInfo : reader.getFieldInfos()) {
+ final String name = fieldInfo.name;
+ allFieldNames.add(name);
+ if (fieldInfo.isIndexed) {
+ indexedFieldNames.add(name);
+ } else {
+ notIndexedFieldNames.add(name);
+ }
+ if (fieldInfo.storeTermVector) {
+ tvFieldNames.add(name);
+ } else if (fieldInfo.isIndexed) {
+ noTVFieldNames.add(name);
+ }
+ }
+
+ assertTrue(allFieldNames.size() == DocHelper.all.size());
+ for (String s : allFieldNames) {
assertTrue(DocHelper.nameValues.containsKey(s) == true || s.equals(""));
}
- result = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
- assertTrue(result != null);
- assertTrue(result.size() == DocHelper.indexed.size());
- for (Iterator<String> iter = result.iterator(); iter.hasNext();) {
- String s = iter.next();
+
+ assertTrue(indexedFieldNames.size() == DocHelper.indexed.size());
+ for (String s : indexedFieldNames) {
assertTrue(DocHelper.indexed.containsKey(s) == true || s.equals(""));
}
- result = reader.getFieldNames(IndexReader.FieldOption.UNINDEXED);
- assertTrue(result != null);
- assertTrue(result.size() == DocHelper.unindexed.size());
+ assertTrue(notIndexedFieldNames.size() == DocHelper.unindexed.size());
//Get all indexed fields that are storing term vectors
- result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR);
- assertTrue(result != null);
- assertTrue(result.size() == DocHelper.termvector.size());
-
- result = reader.getFieldNames(IndexReader.FieldOption.INDEXED_NO_TERMVECTOR);
- assertTrue(result != null);
- assertTrue(result.size() == DocHelper.notermvector.size());
+ assertTrue(tvFieldNames.size() == DocHelper.termvector.size());
+
+ assertTrue(noTVFieldNames.size() == DocHelper.notermvector.size());
}
public void testTerms() throws IOException {
@@ -139,7 +148,8 @@ public class TestSegmentReader extends L
DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(reader,
MultiFields.getLiveDocs(reader),
DocHelper.TEXT_FIELD_1_KEY,
- new BytesRef("field"));
+ new BytesRef("field"),
+ false);
// NOTE: prior rev of this test was failing to first
// call next here:
assertTrue(positions.nextDoc() != DocsEnum.NO_MORE_DOCS);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java Wed Jan 18 22:28:07 2012
@@ -406,7 +406,7 @@ public class TestStressIndexing2 extends
BytesRef term2;
while((term2 = termsEnum3.next()) != null) {
System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq());
- dpEnum = termsEnum3.docsAndPositions(null, dpEnum);
+ dpEnum = termsEnum3.docsAndPositions(null, dpEnum, false);
if (dpEnum != null) {
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
final int freq = dpEnum.freq();
@@ -440,7 +440,7 @@ public class TestStressIndexing2 extends
BytesRef term2;
while((term2 = termsEnum3.next()) != null) {
System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq());
- dpEnum = termsEnum3.docsAndPositions(null, dpEnum);
+ dpEnum = termsEnum3.docsAndPositions(null, dpEnum, false);
if (dpEnum != null) {
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
final int freq = dpEnum.freq();
@@ -630,8 +630,8 @@ public class TestStressIndexing2 extends
assertEquals(termsEnum1.totalTermFreq(),
termsEnum2.totalTermFreq());
- dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1);
- dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2);
+ dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1, false);
+ dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2, false);
if (dpEnum1 != null) {
assertNotNull(dpEnum2);
int docID1 = dpEnum1.nextDoc();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Wed Jan 18 22:28:07 2012
@@ -259,7 +259,7 @@ public class TestTermVectorsReader exten
//System.out.println("Term: " + term);
assertEquals(testTerms[i], term);
- dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
assertNotNull(dpEnum);
int doc = dpEnum.docID();
assertTrue(doc == -1 || doc == DocIdSetIterator.NO_MORE_DOCS);
@@ -270,18 +270,16 @@ public class TestTermVectorsReader exten
}
assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
- dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
doc = dpEnum.docID();
assertTrue(doc == -1 || doc == DocIdSetIterator.NO_MORE_DOCS);
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertNotNull(dpEnum);
- final OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
- assertNotNull(offsetAtt);
assertEquals(dpEnum.freq(), positions[i].length);
for (int j = 0; j < positions[i].length; j++) {
assertEquals(positions[i][j], dpEnum.nextPosition());
- assertEquals(j*10, offsetAtt.startOffset());
- assertEquals(j*10 + testTerms[i].length(), offsetAtt.endOffset());
+ assertEquals(j*10, dpEnum.startOffset());
+ assertEquals(j*10 + testTerms[i].length(), dpEnum.endOffset());
}
assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
}
@@ -315,7 +313,7 @@ public class TestTermVectorsReader exten
String term = text.utf8ToString();
assertEquals(testTerms[i], term);
- dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum, false);
assertNotNull(dpEnum);
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
assertEquals(dpEnum.freq(), positions[i].length);
@@ -324,16 +322,14 @@ public class TestTermVectorsReader exten
}
assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
- dpEnum = termsEnum.docsAndPositions(null, dpEnum);
+ dpEnum = termsEnum.docsAndPositions(null, dpEnum, true);
assertTrue(dpEnum.nextDoc() != DocsEnum.NO_MORE_DOCS);
- final OffsetAttribute offsetAtt = dpEnum.attributes().getAttribute(OffsetAttribute.class);
- assertNotNull(offsetAtt);
assertNotNull(dpEnum);
assertEquals(dpEnum.freq(), positions[i].length);
for (int j = 0; j < positions[i].length; j++) {
assertEquals(positions[i][j], dpEnum.nextPosition());
- assertEquals(j*10, offsetAtt.startOffset());
- assertEquals(j*10 + testTerms[i].length(), offsetAtt.endOffset());
+ assertEquals(j*10, dpEnum.startOffset());
+ assertEquals(j*10 + testTerms[i].length(), dpEnum.endOffset());
}
assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum.nextDoc());
}