You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/10 16:05:21 UTC
svn commit: r1454862 [16/49] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/demo/
dev-tools/idea/lucene/facet/ dev-tools/idea/lucene/highlighter/ dev-to...
Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Sun Mar 10 15:04:57 2013
@@ -16,57 +16,29 @@ package org.apache.lucene.index;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import java.io.Closeable;
+
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.ByteDocValuesField;
-import org.apache.lucene.document.DerefBytesDocValuesField;
+import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.IntDocValuesField;
-import org.apache.lucene.document.LongDocValuesField;
-import org.apache.lucene.document.PackedLongDocValuesField;
-import org.apache.lucene.document.ShortDocValuesField;
-import org.apache.lucene.document.SortedBytesDocValuesField;
-import org.apache.lucene.document.StraightBytesDocValuesField;
-import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DocValues.SortedSource;
-import org.apache.lucene.index.DocValues.Source;
-import org.apache.lucene.index.DocValues.SourceCache;
-import org.apache.lucene.index.DocValues.Type;
-import org.apache.lucene.index.DocValues.SourceCache.DirectSourceCache;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
/**
*
- * Tests DocValues integration into IndexWriter & Codecs
+ * Tests DocValues integration into IndexWriter
*
*/
public class TestDocValuesIndexing extends LuceneTestCase {
@@ -75,70 +47,12 @@ public class TestDocValuesIndexing exten
* - add multithreaded tests / integrate into stress indexing?
*/
- /*
- * Simple test case to show how to use the API
- */
- public void testDocValuesSimple() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, writerConfig(false));
- for (int i = 0; i < 5; i++) {
- Document doc = new Document();
- doc.add(new PackedLongDocValuesField("docId", i));
- doc.add(new TextField("docId", "" + i, Field.Store.NO));
- writer.addDocument(doc);
- }
- writer.commit();
- writer.forceMerge(1, true);
-
- writer.close(true);
-
- DirectoryReader reader = DirectoryReader.open(dir, 1);
- assertEquals(1, reader.leaves().size());
-
- IndexSearcher searcher = new IndexSearcher(reader);
-
- BooleanQuery query = new BooleanQuery();
- query.add(new TermQuery(new Term("docId", "0")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("docId", "1")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("docId", "2")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("docId", "3")), BooleanClause.Occur.SHOULD);
- query.add(new TermQuery(new Term("docId", "4")), BooleanClause.Occur.SHOULD);
-
- TopDocs search = searcher.search(query, 10);
- assertEquals(5, search.totalHits);
- ScoreDoc[] scoreDocs = search.scoreDocs;
- DocValues docValues = MultiDocValues.getDocValues(reader, "docId");
- Source source = docValues.getSource();
- for (int i = 0; i < scoreDocs.length; i++) {
- assertEquals(i, scoreDocs[i].doc);
- assertEquals(i, source.getInt(scoreDocs[i].doc));
- }
- reader.close();
- dir.close();
- }
-
- public void testIndexBytesNoDeletes() throws IOException {
- runTestIndexBytes(writerConfig(random().nextBoolean()), false);
- }
-
- public void testIndexBytesDeletes() throws IOException {
- runTestIndexBytes(writerConfig(random().nextBoolean()), true);
- }
-
- public void testIndexNumericsNoDeletes() throws IOException {
- runTestNumerics(writerConfig(random().nextBoolean()), false);
- }
-
- public void testIndexNumericsDeletes() throws IOException {
- runTestNumerics(writerConfig(random().nextBoolean()), true);
- }
-
public void testAddIndexes() throws IOException {
Directory d1 = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d1);
Document doc = new Document();
doc.add(newStringField("id", "1", Field.Store.YES));
- doc.add(new PackedLongDocValuesField("dv", 1));
+ doc.add(new NumericDocValuesField("dv", 1));
w.addDocument(doc);
IndexReader r1 = w.getReader();
w.close();
@@ -147,7 +61,7 @@ public class TestDocValuesIndexing exten
w = new RandomIndexWriter(random(), d2);
doc = new Document();
doc.add(newStringField("id", "2", Field.Store.YES));
- doc.add(new PackedLongDocValuesField("dv", 2));
+ doc.add(new NumericDocValuesField("dv", 2));
w.addDocument(doc);
IndexReader r2 = w.getReader();
w.close();
@@ -165,601 +79,17 @@ public class TestDocValuesIndexing exten
w.close();
AtomicReader sr = getOnlySegmentReader(r3);
assertEquals(2, sr.numDocs());
- DocValues docValues = sr.docValues("dv");
+ NumericDocValues docValues = sr.getNumericDocValues("dv");
assertNotNull(docValues);
r3.close();
d3.close();
}
- public void testAddIndexesRandom() throws IOException {
- int valuesPerIndex = 10;
- List<Type> values = Arrays.asList(Type.values());
- Collections.shuffle(values, random());
- Type first = values.get(0);
- Type second = values.get(1);
- // index first index
- Directory d_1 = newDirectory();
- IndexWriter w_1 = new IndexWriter(d_1, writerConfig(random().nextBoolean()));
- indexValues(w_1, valuesPerIndex, first, values, false, 7);
- w_1.commit();
- assertEquals(valuesPerIndex, w_1.maxDoc());
- _TestUtil.checkIndex(d_1);
-
- // index second index
- Directory d_2 = newDirectory();
- IndexWriter w_2 = new IndexWriter(d_2, writerConfig(random().nextBoolean()));
- indexValues(w_2, valuesPerIndex, second, values, false, 7);
- w_2.commit();
- assertEquals(valuesPerIndex, w_2.maxDoc());
- _TestUtil.checkIndex(d_2);
-
- Directory target = newDirectory();
- IndexWriter w = new IndexWriter(target, writerConfig(random().nextBoolean()));
- DirectoryReader r_1 = DirectoryReader.open(w_1, true);
- DirectoryReader r_2 = DirectoryReader.open(w_2, true);
- if (random().nextBoolean()) {
- w.addIndexes(d_1, d_2);
- } else {
- w.addIndexes(r_1, r_2);
- }
- w.forceMerge(1, true);
- w.commit();
-
- _TestUtil.checkIndex(target);
- assertEquals(valuesPerIndex * 2, w.maxDoc());
-
- // check values
-
- DirectoryReader merged = DirectoryReader.open(w, true);
- Source source_1 = getSource(getDocValues(r_1, first.name()));
- Source source_2 = getSource(getDocValues(r_2, second.name()));
- Source source_1_merged = getSource(getDocValues(merged, first.name()));
- Source source_2_merged = getSource(getDocValues(merged, second
- .name()));
- for (int i = 0; i < r_1.maxDoc(); i++) {
- switch (first) {
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_SORTED:
- assertEquals(source_1.getBytes(i, new BytesRef()),
- source_1_merged.getBytes(i, new BytesRef()));
- break;
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- case VAR_INTS:
- assertEquals(source_1.getInt(i), source_1_merged.getInt(i));
- break;
- case FLOAT_32:
- case FLOAT_64:
- assertEquals(source_1.getFloat(i), source_1_merged.getFloat(i), 0.0d);
- break;
- default:
- fail("unkonwn " + first);
- }
- }
-
- for (int i = r_1.maxDoc(); i < merged.maxDoc(); i++) {
- switch (second) {
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_SORTED:
- assertEquals(source_2.getBytes(i - r_1.maxDoc(), new BytesRef()),
- source_2_merged.getBytes(i, new BytesRef()));
- break;
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- case VAR_INTS:
- assertEquals(source_2.getInt(i - r_1.maxDoc()),
- source_2_merged.getInt(i));
- break;
- case FLOAT_32:
- case FLOAT_64:
- assertEquals(source_2.getFloat(i - r_1.maxDoc()),
- source_2_merged.getFloat(i), 0.0d);
- break;
- default:
- fail("unkonwn " + first);
- }
- }
- // close resources
- r_1.close();
- r_2.close();
- merged.close();
- w_1.close(true);
- w_2.close(true);
- w.close(true);
- d_1.close();
- d_2.close();
- target.close();
- }
-
- private IndexWriterConfig writerConfig(boolean useCompoundFile) {
- final IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random()));
- cfg.setMergePolicy(newLogMergePolicy(random()));
- LogMergePolicy policy = new LogDocMergePolicy();
- cfg.setMergePolicy(policy);
- policy.setUseCompoundFile(useCompoundFile);
- return cfg;
- }
-
- @SuppressWarnings("fallthrough")
- public void runTestNumerics(IndexWriterConfig cfg, boolean withDeletions)
- throws IOException {
- Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, cfg);
- final int numValues = 50 + atLeast(10);
- final List<Type> numVariantList = new ArrayList<Type>(NUMERICS);
-
- // run in random order to test if fill works correctly during merges
- Collections.shuffle(numVariantList, random());
- for (Type val : numVariantList) {
- FixedBitSet deleted = indexValues(w, numValues, val, numVariantList,
- withDeletions, 7);
- List<Closeable> closeables = new ArrayList<Closeable>();
- DirectoryReader r = DirectoryReader.open(w, true);
- final int numRemainingValues = numValues - deleted.cardinality();
- final int base = r.numDocs() - numRemainingValues;
- // for FIXED_INTS_8 we use value mod 128 - to enable testing in
- // one go we simply use numValues as the mod for all other INT types
- int mod = numValues;
- switch (val) {
- case FIXED_INTS_8:
- mod = 128;
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case VAR_INTS: {
- DocValues intsReader = getDocValues(r, val.name());
- assertNotNull(intsReader);
-
- Source ints = getSource(intsReader);
-
- for (int i = 0; i < base; i++) {
- long value = ints.getInt(i);
- assertEquals("index " + i, 0, value);
- }
-
- int expected = 0;
- for (int i = base; i < r.numDocs(); i++, expected++) {
- while (deleted.get(expected)) {
- expected++;
- }
- assertEquals(val + " mod: " + mod + " index: " + i, expected%mod, ints.getInt(i));
- }
- }
- break;
- case FLOAT_32:
- case FLOAT_64: {
- DocValues floatReader = getDocValues(r, val.name());
- assertNotNull(floatReader);
- Source floats = getSource(floatReader);
- for (int i = 0; i < base; i++) {
- double value = floats.getFloat(i);
- assertEquals(val + " failed for doc: " + i + " base: " + base,
- 0.0d, value, 0.0d);
- }
- int expected = 0;
- for (int i = base; i < r.numDocs(); i++, expected++) {
- while (deleted.get(expected)) {
- expected++;
- }
- assertEquals("index " + i, 2.0 * expected, floats.getFloat(i),
- 0.00001);
- }
- }
- break;
- default:
- fail("unexpected value " + val);
- }
-
- closeables.add(r);
- for (Closeable toClose : closeables) {
- toClose.close();
- }
- }
- w.close();
- d.close();
- }
-
- public void runTestIndexBytes(IndexWriterConfig cfg, boolean withDeletions)
- throws IOException {
- final Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, cfg);
- final List<Type> byteVariantList = new ArrayList<Type>(BYTES);
- // run in random order to test if fill works correctly during merges
- Collections.shuffle(byteVariantList, random());
- final int numValues = 50 + atLeast(10);
- for (Type byteIndexValue : byteVariantList) {
- List<Closeable> closeables = new ArrayList<Closeable>();
- final int bytesSize = 1 + atLeast(50);
- FixedBitSet deleted = indexValues(w, numValues, byteIndexValue,
- byteVariantList, withDeletions, bytesSize);
- final DirectoryReader r = DirectoryReader.open(w, withDeletions);
- assertEquals(0, r.numDeletedDocs());
- final int numRemainingValues = numValues - deleted.cardinality();
- final int base = r.numDocs() - numRemainingValues;
- DocValues bytesReader = getDocValues(r, byteIndexValue.name());
- assertNotNull("field " + byteIndexValue.name()
- + " returned null reader - maybe merged failed", bytesReader);
- Source bytes = getSource(bytesReader);
- byte upto = 0;
-
- // test the filled up slots for correctness
- for (int i = 0; i < base; i++) {
-
- BytesRef br = bytes.getBytes(i, new BytesRef());
- String msg = " field: " + byteIndexValue.name() + " at index: " + i
- + " base: " + base + " numDocs:" + r.numDocs();
- switch (byteIndexValue) {
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
- // fixed straight returns bytesref with zero bytes all of fixed
- // length
- assertNotNull("expected none null - " + msg, br);
- if (br.length != 0) {
- assertEquals("expected zero bytes of length " + bytesSize + " - "
- + msg + br.utf8ToString(), bytesSize, br.length);
- for (int j = 0; j < br.length; j++) {
- assertEquals("Byte at index " + j + " doesn't match - " + msg, 0,
- br.bytes[br.offset + j]);
- }
- }
- break;
- default:
- assertNotNull("expected none null - " + msg, br);
- assertEquals(byteIndexValue + "", 0, br.length);
- // make sure we advance at least until base
- }
- }
-
- // test the actual doc values added in this iteration
- assertEquals(base + numRemainingValues, r.numDocs());
- int v = 0;
- for (int i = base; i < r.numDocs(); i++) {
- String msg = " field: " + byteIndexValue.name() + " at index: " + i
- + " base: " + base + " numDocs:" + r.numDocs() + " bytesSize: "
- + bytesSize + " src: " + bytes;
- while (withDeletions && deleted.get(v++)) {
- upto += bytesSize;
- }
- BytesRef br = bytes.getBytes(i, new BytesRef());
- assertTrue(msg, br.length > 0);
- for (int j = 0; j < br.length; j++, upto++) {
- if (!(br.bytes.length > br.offset + j))
- br = bytes.getBytes(i, new BytesRef());
- assertTrue("BytesRef index exceeded [" + msg + "] offset: "
- + br.offset + " length: " + br.length + " index: "
- + (br.offset + j), br.bytes.length > br.offset + j);
- assertEquals("SourceRef Byte at index " + j + " doesn't match - "
- + msg, upto, br.bytes[br.offset + j]);
- }
- }
-
- // clean up
- closeables.add(r);
- for (Closeable toClose : closeables) {
- toClose.close();
- }
- }
-
- w.close();
- d.close();
- }
-
- public void testGetArrayNumerics() throws IOException {
- Directory d = newDirectory();
- IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter w = new IndexWriter(d, cfg);
- final int numValues = 50 + atLeast(10);
- final List<Type> numVariantList = new ArrayList<Type>(NUMERICS);
- Collections.shuffle(numVariantList, random());
- for (Type val : numVariantList) {
- indexValues(w, numValues, val, numVariantList,
- false, 7);
- DirectoryReader r = DirectoryReader.open(w, true);
- DocValues docValues = getDocValues(r, val.name());
- assertNotNull(docValues);
- // make sure we don't get a direct source since they don't support getArray()
- Source source = docValues.getSource();
- switch (source.getType()) {
- case FIXED_INTS_8:
- {
- assertTrue(source.hasArray());
- byte[] values = (byte[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals((long)values[i], source.getInt(i));
- }
- }
- break;
- case FIXED_INTS_16:
- {
- assertTrue(source.hasArray());
- short[] values = (short[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals((long)values[i], source.getInt(i));
- }
- }
- break;
- case FIXED_INTS_32:
- {
- assertTrue(source.hasArray());
- int[] values = (int[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals((long)values[i], source.getInt(i));
- }
- }
- break;
- case FIXED_INTS_64:
- {
- assertTrue(source.hasArray());
- long[] values = (long[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals(values[i], source.getInt(i));
- }
- }
- break;
- case VAR_INTS:
- assertFalse(source.hasArray());
- break;
- case FLOAT_32:
- {
- assertTrue(source.hasArray());
- float[] values = (float[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals((double)values[i], source.getFloat(i), 0.0d);
- }
- }
- break;
- case FLOAT_64:
- {
- assertTrue(source.hasArray());
- double[] values = (double[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals(values[i], source.getFloat(i), 0.0d);
- }
- }
- break;
- default:
- fail("unexpected value " + source.getType());
- }
- r.close();
- }
- w.close();
- d.close();
- }
-
- public void testGetArrayBytes() throws IOException {
- Directory d = newDirectory();
- IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random()));
- IndexWriter w = new IndexWriter(d, cfg);
- final int numValues = 50 + atLeast(10);
- // only single byte fixed straight supports getArray()
- indexValues(w, numValues, Type.BYTES_FIXED_STRAIGHT, null, false, 1);
- DirectoryReader r = DirectoryReader.open(w, true);
- DocValues docValues = getDocValues(r, Type.BYTES_FIXED_STRAIGHT.name());
- assertNotNull(docValues);
- // make sure we don't get a direct source since they don't support
- // getArray()
- Source source = docValues.getSource();
-
- switch (source.getType()) {
- case BYTES_FIXED_STRAIGHT: {
- BytesRef ref = new BytesRef();
- if (source.hasArray()) {
- byte[] values = (byte[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- source.getBytes(i, ref);
- assertEquals(1, ref.length);
- assertEquals(values[i], ref.bytes[ref.offset]);
- }
- }
- }
- break;
- default:
- fail("unexpected value " + source.getType());
- }
- r.close();
- w.close();
- d.close();
- }
-
- private DocValues getDocValues(IndexReader reader, String field) throws IOException {
- final DocValues docValues = MultiDocValues.getDocValues(reader, field);
- if (docValues == null) {
- return docValues;
- }
- if (rarely()) {
- docValues.setCache(new NotCachingSourceCache());
- } else {
- if (!(docValues.getCache() instanceof DirectSourceCache)) {
- docValues.setCache(new DirectSourceCache());
- }
- }
- return docValues;
- }
-
- @SuppressWarnings("fallthrough")
- private Source getSource(DocValues values) throws IOException {
- // getSource uses cache internally
- switch(random().nextInt(5)) {
- case 3:
- return values.loadSource();
- case 2:
- return values.getDirectSource();
- case 1:
- if(values.getType() == Type.BYTES_VAR_SORTED || values.getType() == Type.BYTES_FIXED_SORTED) {
- return values.getSource().asSortedSource();
- }
- default:
- return values.getSource();
- }
- }
-
-
- private static EnumSet<Type> BYTES = EnumSet.of(Type.BYTES_FIXED_DEREF,
- Type.BYTES_FIXED_STRAIGHT, Type.BYTES_VAR_DEREF,
- Type.BYTES_VAR_STRAIGHT, Type.BYTES_FIXED_SORTED, Type.BYTES_VAR_SORTED);
-
- private static EnumSet<Type> NUMERICS = EnumSet.of(Type.VAR_INTS,
- Type.FIXED_INTS_16, Type.FIXED_INTS_32,
- Type.FIXED_INTS_64,
- Type.FIXED_INTS_8,
- Type.FLOAT_32,
- Type.FLOAT_64);
-
- private FixedBitSet indexValues(IndexWriter w, int numValues, Type valueType,
- List<Type> valueVarList, boolean withDeletions, int bytesSize)
- throws IOException {
- final boolean isNumeric = NUMERICS.contains(valueType);
- FixedBitSet deleted = new FixedBitSet(numValues);
- Document doc = new Document();
- final Field valField;
- if (isNumeric) {
- switch (valueType) {
- case VAR_INTS:
- valField = new PackedLongDocValuesField(valueType.name(), (long) 0);
- break;
- case FIXED_INTS_16:
- valField = new ShortDocValuesField(valueType.name(), (short) 0);
- break;
- case FIXED_INTS_32:
- valField = new IntDocValuesField(valueType.name(), 0);
- break;
- case FIXED_INTS_64:
- valField = new LongDocValuesField(valueType.name(), (long) 0);
- break;
- case FIXED_INTS_8:
- valField = new ByteDocValuesField(valueType.name(), (byte) 0);
- break;
- case FLOAT_32:
- valField = new FloatDocValuesField(valueType.name(), (float) 0);
- break;
- case FLOAT_64:
- valField = new DoubleDocValuesField(valueType.name(), (double) 0);
- break;
- default:
- valField = null;
- fail("unhandled case");
- }
- } else {
- switch (valueType) {
- case BYTES_FIXED_STRAIGHT:
- valField = new StraightBytesDocValuesField(valueType.name(), new BytesRef(), true);
- break;
- case BYTES_VAR_STRAIGHT:
- valField = new StraightBytesDocValuesField(valueType.name(), new BytesRef(), false);
- break;
- case BYTES_FIXED_DEREF:
- valField = new DerefBytesDocValuesField(valueType.name(), new BytesRef(), true);
- break;
- case BYTES_VAR_DEREF:
- valField = new DerefBytesDocValuesField(valueType.name(), new BytesRef(), false);
- break;
- case BYTES_FIXED_SORTED:
- valField = new SortedBytesDocValuesField(valueType.name(), new BytesRef(), true);
- break;
- case BYTES_VAR_SORTED:
- valField = new SortedBytesDocValuesField(valueType.name(), new BytesRef(), false);
- break;
- default:
- valField = null;
- fail("unhandled case");
- }
- }
- doc.add(valField);
- final BytesRef bytesRef = new BytesRef();
-
- final String idBase = valueType.name() + "_";
- final byte[] b = new byte[bytesSize];
- if (bytesRef != null) {
- bytesRef.bytes = b;
- bytesRef.length = b.length;
- bytesRef.offset = 0;
- }
- byte upto = 0;
- for (int i = 0; i < numValues; i++) {
- if (isNumeric) {
- switch (valueType) {
- case VAR_INTS:
- valField.setLongValue((long)i);
- break;
- case FIXED_INTS_16:
- valField.setShortValue((short)i);
- break;
- case FIXED_INTS_32:
- valField.setIntValue(i);
- break;
- case FIXED_INTS_64:
- valField.setLongValue((long)i);
- break;
- case FIXED_INTS_8:
- valField.setByteValue((byte)(0xFF & (i % 128)));
- break;
- case FLOAT_32:
- valField.setFloatValue(2.0f * i);
- break;
- case FLOAT_64:
- valField.setDoubleValue(2.0d * i);
- break;
- default:
- fail("unexpected value " + valueType);
- }
- } else {
- for (int j = 0; j < b.length; j++) {
- b[j] = upto++;
- }
- if (bytesRef != null) {
- valField.setBytesValue(bytesRef);
- }
- }
- doc.removeFields("id");
- doc.add(new StringField("id", idBase + i, Field.Store.YES));
- w.addDocument(doc);
-
- if (i % 7 == 0) {
- if (withDeletions && random().nextBoolean()) {
- Type val = valueVarList.get(random().nextInt(1 + valueVarList
- .indexOf(valueType)));
- final int randInt = val == valueType ? random().nextInt(1 + i) : random()
- .nextInt(numValues);
- w.deleteDocuments(new Term("id", val.name() + "_" + randInt));
- if (val == valueType) {
- deleted.set(randInt);
- }
- }
- if (random().nextInt(10) == 0) {
- w.commit();
- }
- }
- }
- w.commit();
-
- // TODO test multi seg with deletions
- if (withDeletions || random().nextBoolean()) {
- w.forceMerge(1, true);
- }
- return deleted;
- }
-
public void testMultiValuedDocValuesField() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
Document doc = new Document();
- Field f = new PackedLongDocValuesField("field", 17);
+ Field f = new NumericDocValuesField("field", 17);
// Index doc values are single-valued so we should not
// be able to add same field more than once:
doc.add(f);
@@ -777,7 +107,7 @@ public class TestDocValuesIndexing exten
w.forceMerge(1);
DirectoryReader r = w.getReader();
w.close();
- assertEquals(17, getOnlySegmentReader(r).docValues("field").loadSource().getInt(0));
+ assertEquals(17, FieldCache.DEFAULT.getInts(getOnlySegmentReader(r), "field", false).get(0));
r.close();
d.close();
}
@@ -789,8 +119,8 @@ public class TestDocValuesIndexing exten
// Index doc values are single-valued so we should not
// be able to add same field more than once:
Field f;
- doc.add(f = new PackedLongDocValuesField("field", 17));
- doc.add(new FloatDocValuesField("field", 22.0f));
+ doc.add(f = new NumericDocValuesField("field", 17));
+ doc.add(new BinaryDocValuesField("field", new BytesRef("blah")));
try {
w.addDocument(doc);
fail("didn't hit expected exception");
@@ -804,208 +134,35 @@ public class TestDocValuesIndexing exten
w.forceMerge(1);
DirectoryReader r = w.getReader();
w.close();
- assertEquals(17, getOnlySegmentReader(r).docValues("field").loadSource().getInt(0));
+ assertEquals(17, FieldCache.DEFAULT.getInts(getOnlySegmentReader(r), "field", false).get(0));
r.close();
d.close();
}
-
- public void testSortedBytes() throws IOException {
- Type[] types = new Type[] { Type.BYTES_FIXED_SORTED, Type.BYTES_VAR_SORTED };
- for (Type type : types) {
- boolean fixed = type == Type.BYTES_FIXED_SORTED;
- final Directory d = newDirectory();
- IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random()));
- IndexWriter w = new IndexWriter(d, cfg);
- int numDocs = atLeast(100);
- BytesRefHash hash = new BytesRefHash();
- Map<String, String> docToString = new HashMap<String, String>();
- int len = 1 + random().nextInt(50);
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- doc.add(newTextField("id", "" + i, Field.Store.YES));
- String string = fixed ? _TestUtil.randomFixedByteLengthUnicodeString(random(),
- len) : _TestUtil.randomRealisticUnicodeString(random(), 1, len);
- BytesRef br = new BytesRef(string);
- doc.add(new SortedBytesDocValuesField("field", br, type == Type.BYTES_FIXED_SORTED));
- hash.add(br);
- docToString.put("" + i, string);
- w.addDocument(doc);
- }
- if (rarely()) {
- w.commit();
- }
- int numDocsNoValue = atLeast(10);
- for (int i = 0; i < numDocsNoValue; i++) {
- Document doc = new Document();
- doc.add(newTextField("id", "noValue", Field.Store.YES));
- w.addDocument(doc);
- }
- BytesRef bytesRef = new BytesRef(fixed ? len : 0);
- bytesRef.offset = 0;
- bytesRef.length = fixed ? len : 0;
- hash.add(bytesRef); // add empty value for the gaps
- if (rarely()) {
- w.commit();
- }
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- String id = "" + i + numDocs;
- doc.add(newTextField("id", id, Field.Store.YES));
- String string = fixed ? _TestUtil.randomFixedByteLengthUnicodeString(random(),
- len) : _TestUtil.randomRealisticUnicodeString(random(), 1, len);
- BytesRef br = new BytesRef(string);
- hash.add(br);
- docToString.put(id, string);
- doc.add(new SortedBytesDocValuesField("field", br, type == Type.BYTES_FIXED_SORTED));
- w.addDocument(doc);
- }
- w.commit();
- IndexReader reader = w.getReader();
- DocValues docValues = MultiDocValues.getDocValues(reader, "field");
- Source source = getSource(docValues);
- SortedSource asSortedSource = source.asSortedSource();
- int[] sort = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
- BytesRef expected = new BytesRef();
- BytesRef actual = new BytesRef();
- assertEquals(hash.size(), asSortedSource.getValueCount());
- for (int i = 0; i < hash.size(); i++) {
- hash.get(sort[i], expected);
- asSortedSource.getByOrd(i, actual);
- assertEquals(expected.utf8ToString(), actual.utf8ToString());
- int ord = asSortedSource.getOrdByValue(expected, actual);
- assertEquals(i, ord);
- }
- AtomicReader slowR = SlowCompositeReaderWrapper.wrap(reader);
- Set<Entry<String, String>> entrySet = docToString.entrySet();
- for (Entry<String, String> entry : entrySet) {
- int docId = docId(slowR, new Term("id", entry.getKey()));
- expected = new BytesRef(entry.getValue());
- assertEquals(expected, asSortedSource.getBytes(docId, actual));
- }
-
- reader.close();
- w.close();
- d.close();
- }
- }
-
- public int docId(AtomicReader reader, Term term) throws IOException {
- int docFreq = reader.docFreq(term);
- assertEquals(1, docFreq);
- DocsEnum termDocsEnum = reader.termDocsEnum(term);
- int nextDoc = termDocsEnum.nextDoc();
- assertEquals(DocIdSetIterator.NO_MORE_DOCS, termDocsEnum.nextDoc());
- return nextDoc;
- }
-
- public void testWithThreads() throws Exception {
- Random random = random();
- final int NUM_DOCS = atLeast(100);
- final Directory dir = newDirectory();
- final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
- final boolean allowDups = random.nextBoolean();
- final Set<String> seen = new HashSet<String>();
- if (VERBOSE) {
- System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " allowDups=" + allowDups);
- }
- int numDocs = 0;
- final List<BytesRef> docValues = new ArrayList<BytesRef>();
-
- // TODO: deletions
- while (numDocs < NUM_DOCS) {
- final String s;
- if (random.nextBoolean()) {
- s = _TestUtil.randomSimpleString(random);
- } else {
- s = _TestUtil.randomUnicodeString(random);
- }
- final BytesRef br = new BytesRef(s);
-
- if (!allowDups) {
- if (seen.contains(s)) {
- continue;
- }
- seen.add(s);
- }
-
- if (VERBOSE) {
- System.out.println(" " + numDocs + ": s=" + s);
- }
-
- final Document doc = new Document();
- doc.add(new SortedBytesDocValuesField("stringdv", br));
- doc.add(new PackedLongDocValuesField("id", numDocs));
- docValues.add(br);
- writer.addDocument(doc);
- numDocs++;
-
- if (random.nextInt(40) == 17) {
- // force flush
- writer.getReader().close();
- }
- }
-
- writer.forceMerge(1);
- final DirectoryReader r = writer.getReader();
- writer.close();
-
- final AtomicReader sr = getOnlySegmentReader(r);
- final DocValues dv = sr.docValues("stringdv");
- assertNotNull(dv);
-
- final long END_TIME = System.currentTimeMillis() + (TEST_NIGHTLY ? 30 : 1);
-
- final DocValues.Source docIDToID = sr.docValues("id").getSource();
-
- final int NUM_THREADS = _TestUtil.nextInt(random(), 1, 10);
- Thread[] threads = new Thread[NUM_THREADS];
- for(int thread=0;thread<NUM_THREADS;thread++) {
- threads[thread] = new Thread() {
- @Override
- public void run() {
- Random random = random();
- final DocValues.Source stringDVSource;
- final DocValues.Source stringDVDirectSource;
- try {
- stringDVSource = dv.getSource();
- assertNotNull(stringDVSource);
- stringDVDirectSource = dv.getDirectSource();
- assertNotNull(stringDVDirectSource);
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- while(System.currentTimeMillis() < END_TIME) {
- final DocValues.Source source;
- if (random.nextBoolean()) {
- source = stringDVSource;
- } else {
- source = stringDVDirectSource;
- }
-
- final DocValues.SortedSource sortedSource = source.asSortedSource();
- assertNotNull(sortedSource);
-
- final BytesRef scratch = new BytesRef();
-
- for(int iter=0;iter<100;iter++) {
- final int docID = random.nextInt(sr.maxDoc());
- final BytesRef br = sortedSource.getBytes(docID, scratch);
- assertEquals(docValues.get((int) docIDToID.getInt(docID)), br);
- }
- }
- }
- };
- threads[thread].start();
- }
-
- for(Thread thread : threads) {
- thread.join();
+ public void testDifferentTypedDocValuesField2() throws Exception {
+ Directory d = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), d);
+ Document doc = new Document();
+ // Index doc values are single-valued so we should not
+ // be able to add same field more than once:
+ Field f = new NumericDocValuesField("field", 17);
+ doc.add(f);
+ doc.add(new SortedDocValuesField("field", new BytesRef("hello")));
+ try {
+ w.addDocument(doc);
+ fail("didn't hit expected exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
}
-
+ doc = new Document();
+ doc.add(f);
+ w.addDocument(doc);
+ w.forceMerge(1);
+ DirectoryReader r = w.getReader();
+ assertEquals(17, getOnlySegmentReader(r).getNumericDocValues("field").get(0));
r.close();
- dir.close();
+ w.close();
+ d.close();
}
// LUCENE-3870
@@ -1017,20 +174,21 @@ public class TestDocValuesIndexing exten
BytesRef b = new BytesRef();
b.bytes = bytes;
b.length = bytes.length;
- doc.add(new DerefBytesDocValuesField("field", b));
+ doc.add(new SortedDocValuesField("field", b));
w.addDocument(doc);
bytes[0] = 1;
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = w.getReader();
- Source s = getOnlySegmentReader(r).docValues("field").getSource();
+ BinaryDocValues s = FieldCache.DEFAULT.getTerms(getOnlySegmentReader(r), "field");
- BytesRef bytes1 = s.getBytes(0, new BytesRef());
+ BytesRef bytes1 = new BytesRef();
+ s.get(0, bytes1);
assertEquals(bytes.length, bytes1.length);
bytes[0] = 0;
assertEquals(b, bytes1);
- bytes1 = s.getBytes(1, new BytesRef());
+ s.get(1, bytes1);
assertEquals(bytes.length, bytes1.length);
bytes[0] = 1;
assertEquals(b, bytes1);
@@ -1038,24 +196,7 @@ public class TestDocValuesIndexing exten
w.close();
d.close();
}
-
- public void testFixedLengthNotReallyFixed() throws IOException {
- Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
- Document doc = new Document();
- doc.add(new DerefBytesDocValuesField("foo", new BytesRef("bar"), true));
- w.addDocument(doc);
- doc = new Document();
- doc.add(new DerefBytesDocValuesField("foo", new BytesRef("bazz"), true));
- try {
- w.addDocument(doc);
- } catch (IllegalArgumentException expected) {
- // expected
- }
- w.close();
- d.close();
- }
-
+
public void testDocValuesUnstored() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwconfig = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
@@ -1063,7 +204,7 @@ public class TestDocValuesIndexing exten
IndexWriter writer = new IndexWriter(dir, iwconfig);
for (int i = 0; i < 50; i++) {
Document doc = new Document();
- doc.add(new PackedLongDocValuesField("dv", i));
+ doc.add(new NumericDocValuesField("dv", i));
doc.add(new TextField("docId", "" + i, Field.Store.YES));
writer.addDocument(doc);
}
@@ -1072,10 +213,9 @@ public class TestDocValuesIndexing exten
FieldInfos fi = slow.getFieldInfos();
FieldInfo dvInfo = fi.fieldInfo("dv");
assertTrue(dvInfo.hasDocValues());
- DocValues dv = slow.docValues("dv");
- Source source = dv.getDirectSource();
+ NumericDocValues dv = slow.getNumericDocValues("dv");
for (int i = 0; i < 50; i++) {
- assertEquals(i, source.getInt(i));
+ assertEquals(i, dv.get(i));
StoredDocument d = slow.document(i);
// cannot use d.get("dv") due to another bug!
assertNull(d.getField("dv"));
@@ -1085,24 +225,554 @@ public class TestDocValuesIndexing exten
writer.close();
dir.close();
}
+
+ // Same field in one document as different types:
+ public void testMixedTypesSameDocument() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 0));
+ doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+ try {
+ w.addDocument(doc);
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ w.close();
+ dir.close();
+ }
+
+ // Two documents with same field as different types:
+ public void testMixedTypesDifferentDocuments() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 0));
+ w.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+ try {
+ w.addDocument(doc);
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ w.close();
+ dir.close();
+ }
+
+ public void testAddSortedTwice() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo!")));
+ doc.add(new SortedDocValuesField("dv", new BytesRef("bar!")));
+ try {
+ iwriter.addDocument(doc);
+ fail("didn't hit expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+
+ iwriter.close();
+ directory.close();
+ }
+
+ public void testAddBinaryTwice() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("foo!")));
+ doc.add(new BinaryDocValuesField("dv", new BytesRef("bar!")));
+ try {
+ iwriter.addDocument(doc);
+ fail("didn't hit expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+
+ iwriter.close();
+ directory.close();
+ }
+
+ public void testAddNumericTwice() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 1));
+ doc.add(new NumericDocValuesField("dv", 2));
+ try {
+ iwriter.addDocument(doc);
+ fail("didn't hit expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+
+ iwriter.close();
+ directory.close();
+ }
- /**
- *
- */
- public static class NotCachingSourceCache extends SourceCache {
-
- @Override
- public Source load(DocValues values) throws IOException {
- return values.loadSource();
- }
-
- @Override
- public Source loadDirect(DocValues values) throws IOException {
- return values.loadDirectSource();
- }
-
- @Override
- public void invalidate(DocValues values) {}
- }
-
+ public void testTooLargeBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ byte bytes[] = new byte[100000];
+ BytesRef b = new BytesRef(bytes);
+ random().nextBytes(bytes);
+ doc.add(new BinaryDocValuesField("dv", b));
+ try {
+ iwriter.addDocument(doc);
+ fail("did not get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ iwriter.close();
+
+ directory.close();
+ }
+
+ public void testTooLargeSortedBytes() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ byte bytes[] = new byte[100000];
+ BytesRef b = new BytesRef(bytes);
+ random().nextBytes(bytes);
+ doc.add(new SortedDocValuesField("dv", b));
+ try {
+ iwriter.addDocument(doc);
+ fail("did not get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ iwriter.close();
+ directory.close();
+ }
+
+ public void testTooLargeTermSortedSetBytes() throws IOException {
+ assumeTrue("codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ Directory directory = newDirectory();
+ // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwc.setMergePolicy(newLogMergePolicy());
+ IndexWriter iwriter = new IndexWriter(directory, iwc);
+ Document doc = new Document();
+ byte bytes[] = new byte[100000];
+ BytesRef b = new BytesRef(bytes);
+ random().nextBytes(bytes);
+ doc.add(new SortedSetDocValuesField("dv", b));
+ try {
+ iwriter.addDocument(doc);
+ fail("did not get expected exception");
+ } catch (IllegalArgumentException expected) {
+ // expected
+ }
+ iwriter.close();
+ directory.close();
+ }
+
+ // Two documents across segments
+ public void testMixedTypesDifferentSegments() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 0));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+ try {
+ w.addDocument(doc);
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ w.close();
+ dir.close();
+ }
+
+ // Add inconsistent document after deleteAll
+ public void testMixedTypesAfterDeleteAll() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 0));
+ w.addDocument(doc);
+ w.deleteAll();
+
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+ w.addDocument(doc);
+ w.close();
+ dir.close();
+ }
+
+ // Add inconsistent document after reopening IW w/ create
+ public void testMixedTypesAfterReopenCreate() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 0));
+ w.addDocument(doc);
+ w.close();
+
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
+ w = new IndexWriter(dir, iwc);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+ w.addDocument(doc);
+ w.close();
+ dir.close();
+ }
+
+ // Two documents with same field as different types, added
+ // from separate threads:
+ public void testMixedTypesDifferentThreads() throws Exception {
+ Directory dir = newDirectory();
+ final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+
+ final CountDownLatch startingGun = new CountDownLatch(1);
+ final AtomicBoolean hitExc = new AtomicBoolean();
+ Thread[] threads = new Thread[3];
+ for(int i=0;i<3;i++) {
+ Field field;
+ if (i == 0) {
+ field = new SortedDocValuesField("foo", new BytesRef("hello"));
+ } else if (i == 1) {
+ field = new NumericDocValuesField("foo", 0);
+ } else {
+ field = new BinaryDocValuesField("foo", new BytesRef("bazz"));
+ }
+ final Document doc = new Document();
+ doc.add(field);
+
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ try {
+ startingGun.await();
+ w.addDocument(doc);
+ } catch (IllegalArgumentException iae) {
+ // expected
+ hitExc.set(true);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ threads[i].start();
+ }
+
+ startingGun.countDown();
+
+ for(Thread t : threads) {
+ t.join();
+ }
+ assertTrue(hitExc.get());
+ w.close();
+ dir.close();
+ }
+
+ // Adding documents via addIndexes
+ public void testMixedTypesViaAddIndexes() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 0));
+ w.addDocument(doc);
+
+ // Make 2nd index w/ inconsistent field
+ Directory dir2 = newDirectory();
+ IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("hello")));
+ w2.addDocument(doc);
+ w2.close();
+
+ try {
+ w.addIndexes(new Directory[] {dir2});
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+
+ IndexReader r = DirectoryReader.open(dir2);
+ try {
+ w.addIndexes(new IndexReader[] {r});
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+
+ r.close();
+ dir2.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testIllegalTypeChange() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ try {
+ writer.addDocument(doc);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ writer.close();
+ dir.close();
+ }
+
+ public void testIllegalTypeChangeAcrossSegments() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.close();
+
+ writer = new IndexWriter(dir, conf);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ try {
+ writer.addDocument(doc);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ writer.close();
+ dir.close();
+ }
+
+ public void testTypeChangeAfterCloseAndDeleteAll() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.close();
+
+ writer = new IndexWriter(dir, conf);
+ writer.deleteAll();
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ writer.addDocument(doc);
+ writer.close();
+ dir.close();
+ }
+
+ public void testTypeChangeAfterDeleteAll() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.deleteAll();
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ writer.addDocument(doc);
+ writer.close();
+ dir.close();
+ }
+
+ public void testTypeChangeAfterCommitAndDeleteAll() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.commit();
+ writer.deleteAll();
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ writer.addDocument(doc);
+ writer.close();
+ dir.close();
+ }
+
+ public void testTypeChangeAfterOpenCreate() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.close();
+ conf.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
+ writer = new IndexWriter(dir, conf);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ writer.addDocument(doc);
+ writer.close();
+ dir.close();
+ }
+
+ public void testTypeChangeViaAddIndexes() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.close();
+
+ Directory dir2 = newDirectory();
+ writer = new IndexWriter(dir2, conf);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ writer.addDocument(doc);
+ try {
+ writer.addIndexes(dir);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ writer.close();
+
+ dir.close();
+ dir2.close();
+ }
+
+ public void testTypeChangeViaAddIndexesIR() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.close();
+
+ Directory dir2 = newDirectory();
+ writer = new IndexWriter(dir2, conf);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ writer.addDocument(doc);
+ IndexReader[] readers = new IndexReader[] {DirectoryReader.open(dir)};
+ try {
+ writer.addIndexes(readers);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ readers[0].close();
+ writer.close();
+
+ dir.close();
+ dir2.close();
+ }
+
+ public void testTypeChangeViaAddIndexes2() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.close();
+
+ Directory dir2 = newDirectory();
+ writer = new IndexWriter(dir2, conf);
+ writer.addIndexes(dir);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ try {
+ writer.addDocument(doc);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ writer.close();
+ dir2.close();
+ dir.close();
+ }
+
+ public void testTypeChangeViaAddIndexesIR2() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+ writer.close();
+
+ Directory dir2 = newDirectory();
+ writer = new IndexWriter(dir2, conf);
+ IndexReader[] readers = new IndexReader[] {DirectoryReader.open(dir)};
+ writer.addIndexes(readers);
+ readers[0].close();
+ doc = new Document();
+ doc.add(new SortedDocValuesField("dv", new BytesRef("foo")));
+ try {
+ writer.addDocument(doc);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ writer.close();
+ dir2.close();
+ dir.close();
+ }
+
+ public void testDocsWithField() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new TextField("dv", "some text", Field.Store.NO));
+ doc.add(new NumericDocValuesField("dv", 0L));
+ writer.addDocument(doc);
+
+ DirectoryReader r = writer.getReader();
+ writer.close();
+
+ AtomicReader subR = r.leaves().get(0).reader();
+ assertEquals(2, subR.numDocs());
+
+ Bits bits = FieldCache.DEFAULT.getDocsWithField(subR, "dv");
+ assertTrue(bits.get(0));
+ assertTrue(bits.get(1));
+ r.close();
+ dir.close();
+ }
+
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java Sun Mar 10 15:04:57 2013
@@ -97,7 +97,7 @@ public class TestDocumentWriter extends
// omitNorms is true
for (FieldInfo fi : reader.getFieldInfos()) {
if (fi.isIndexed()) {
- assertTrue(fi.omitsNorms() == (reader.normValues(fi.name) == null));
+ assertTrue(fi.omitsNorms() == (reader.getNormValues(fi.name) == null));
}
}
reader.close();
Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Sun Mar 10 15:04:57 2013
@@ -31,6 +31,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
@@ -39,6 +40,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp;
@@ -61,16 +63,13 @@ public class TestDuelingCodecs extends L
public void setUp() throws Exception {
super.setUp();
- // for now its SimpleText vs Lucene41(random postings format)
+ // for now its SimpleText vs Lucene42(random postings format)
// as this gives the best overall coverage. when we have more
// codecs we should probably pick 2 from Codec.availableCodecs()
- // TODO: it would also be nice to support preflex, but it doesn't
- // support a lot of the current feature set (docvalues, statistics)
- // so this would make assertEquals complicated.
-
leftCodec = Codec.forName("SimpleText");
rightCodec = new RandomCodec(random());
+
leftDir = newDirectory();
rightDir = newDirectory();
@@ -105,15 +104,28 @@ public class TestDuelingCodecs extends L
rightReader = maybeWrapReader(rightWriter.getReader());
rightWriter.close();
+ // check that our readers are valid
+ _TestUtil.checkReader(leftReader);
+ _TestUtil.checkReader(rightReader);
+
info = "left: " + leftCodec.toString() + " / right: " + rightCodec.toString();
}
@Override
public void tearDown() throws Exception {
- leftReader.close();
- rightReader.close();
- leftDir.close();
- rightDir.close();
+ if (leftReader != null) {
+ leftReader.close();
+ }
+ if (rightReader != null) {
+ rightReader.close();
+ }
+
+ if (leftDir != null) {
+ leftDir.close();
+ }
+ if (rightDir != null) {
+ rightDir.close();
+ }
super.tearDown();
}
@@ -129,7 +141,14 @@ public class TestDuelingCodecs extends L
// TODO: we should add other fields that use things like docs&freqs but omit positions,
// because linefiledocs doesn't cover all the possibilities.
for (int i = 0; i < numdocs; i++) {
- writer.addDocument(lineFileDocs.nextDoc());
+ Document document = lineFileDocs.nextDoc();
+ // grab the title and add some SortedSet instances for fun
+ String title = document.get("titleTokenized");
+ String split[] = title.split("\\s+");
+ for (String trash : split) {
+ document.add(new SortedSetDocValuesField("sortedset", new BytesRef(trash)));
+ }
+ writer.addDocument(document);
}
lineFileDocs.close();
@@ -231,11 +250,11 @@ public class TestDuelingCodecs extends L
private void assertTermsSeeking(Terms leftTerms, Terms rightTerms) throws Exception {
TermsEnum leftEnum = null;
TermsEnum rightEnum = null;
-
+
// just an upper bound
int numTests = atLeast(20);
Random random = random();
-
+
// collect this number of terms from the left side
HashSet<BytesRef> tests = new HashSet<BytesRef>();
int numPasses = 0;
@@ -259,36 +278,50 @@ public class TestDuelingCodecs extends L
byte newbytes[] = new byte[term.length+5];
System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length);
tests.add(new BytesRef(newbytes, 5, term.length));
+ } else if (code == 3) {
+ switch (random().nextInt(3)) {
+ case 0:
+ tests.add(new BytesRef()); // before the first term
+ break;
+ case 1:
+ tests.add(new BytesRef(new byte[] {(byte) 0xFF, (byte) 0xFF})); // past the last term
+ break;
+ case 2:
+ tests.add(new BytesRef(_TestUtil.randomSimpleString(random()))); // random term
+ break;
+ default:
+ throw new AssertionError();
+ }
}
}
numPasses++;
}
-
+
+ rightEnum = rightTerms.iterator(rightEnum);
+
ArrayList<BytesRef> shuffledTests = new ArrayList<BytesRef>(tests);
Collections.shuffle(shuffledTests, random);
-
+
for (BytesRef b : shuffledTests) {
- leftEnum = leftTerms.iterator(leftEnum);
- rightEnum = rightTerms.iterator(rightEnum);
-
- assertEquals(info, leftEnum.seekExact(b, false), rightEnum.seekExact(b, false));
- assertEquals(info, leftEnum.seekExact(b, true), rightEnum.seekExact(b, true));
-
- SeekStatus leftStatus;
- SeekStatus rightStatus;
-
- leftStatus = leftEnum.seekCeil(b, false);
- rightStatus = rightEnum.seekCeil(b, false);
- assertEquals(info, leftStatus, rightStatus);
- if (leftStatus != SeekStatus.END) {
- assertEquals(info, leftEnum.term(), rightEnum.term());
+ if (rarely()) {
+ // reuse the enums
+ leftEnum = leftTerms.iterator(leftEnum);
+ rightEnum = rightTerms.iterator(rightEnum);
}
-
- leftStatus = leftEnum.seekCeil(b, true);
- rightStatus = rightEnum.seekCeil(b, true);
- assertEquals(info, leftStatus, rightStatus);
- if (leftStatus != SeekStatus.END) {
- assertEquals(info, leftEnum.term(), rightEnum.term());
+
+ final boolean useCache = random().nextBoolean();
+ final boolean seekExact = random().nextBoolean();
+
+ if (seekExact) {
+ assertEquals(info, leftEnum.seekExact(b, useCache), rightEnum.seekExact(b, useCache));
+ } else {
+ SeekStatus leftStatus = leftEnum.seekCeil(b, useCache);
+ SeekStatus rightStatus = rightEnum.seekCeil(b, useCache);
+ assertEquals(info, leftStatus, rightStatus);
+ if (leftStatus != SeekStatus.END) {
+ assertEquals(info, leftEnum.term(), rightEnum.term());
+ assertTermStats(leftEnum, rightEnum);
+ }
}
}
}
@@ -522,13 +555,13 @@ public class TestDuelingCodecs extends L
}
for (String field : leftFields) {
- DocValues leftNorms = MultiDocValues.getNormDocValues(leftReader, field);
- DocValues rightNorms = MultiDocValues.getNormDocValues(rightReader, field);
+ NumericDocValues leftNorms = MultiDocValues.getNormValues(leftReader, field);
+ NumericDocValues rightNorms = MultiDocValues.getNormValues(rightReader, field);
if (leftNorms != null && rightNorms != null) {
- assertDocValues(leftNorms, rightNorms);
+ assertDocValues(leftReader.maxDoc(), leftNorms, rightNorms);
} else {
- assertNull(leftNorms);
- assertNull(rightNorms);
+ assertNull(info, leftNorms);
+ assertNull(info, rightNorms);
}
}
}
@@ -604,68 +637,104 @@ public class TestDuelingCodecs extends L
* checks that docvalues across all fields are equivalent
*/
public void assertDocValues(IndexReader leftReader, IndexReader rightReader) throws Exception {
- Set<String> leftValues = getDVFields(leftReader);
- Set<String> rightValues = getDVFields(rightReader);
- assertEquals(info, leftValues, rightValues);
-
- for (String field : leftValues) {
- DocValues leftDocValues = MultiDocValues.getDocValues(leftReader, field);
- DocValues rightDocValues = MultiDocValues.getDocValues(rightReader, field);
- if (leftDocValues != null && rightDocValues != null) {
- assertDocValues(leftDocValues, rightDocValues);
- } else {
- assertNull(leftDocValues);
- assertNull(rightDocValues);
+ Set<String> leftFields = getDVFields(leftReader);
+ Set<String> rightFields = getDVFields(rightReader);
+ assertEquals(info, leftFields, rightFields);
+
+ for (String field : leftFields) {
+ // TODO: clean this up... very messy
+ {
+ NumericDocValues leftValues = MultiDocValues.getNumericValues(leftReader, field);
+ NumericDocValues rightValues = MultiDocValues.getNumericValues(rightReader, field);
+ if (leftValues != null && rightValues != null) {
+ assertDocValues(leftReader.maxDoc(), leftValues, rightValues);
+ } else {
+ assertNull(info, leftValues);
+ assertNull(info, rightValues);
+ }
+ }
+
+ {
+ BinaryDocValues leftValues = MultiDocValues.getBinaryValues(leftReader, field);
+ BinaryDocValues rightValues = MultiDocValues.getBinaryValues(rightReader, field);
+ if (leftValues != null && rightValues != null) {
+ BytesRef scratchLeft = new BytesRef();
+ BytesRef scratchRight = new BytesRef();
+ for(int docID=0;docID<leftReader.maxDoc();docID++) {
+ leftValues.get(docID, scratchLeft);
+ rightValues.get(docID, scratchRight);
+ assertEquals(info, scratchLeft, scratchRight);
+ }
+ } else {
+ assertNull(info, leftValues);
+ assertNull(info, rightValues);
+ }
+ }
+
+ {
+ SortedDocValues leftValues = MultiDocValues.getSortedValues(leftReader, field);
+ SortedDocValues rightValues = MultiDocValues.getSortedValues(rightReader, field);
+ if (leftValues != null && rightValues != null) {
+ // numOrds
+ assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
+ // ords
+ BytesRef scratchLeft = new BytesRef();
+ BytesRef scratchRight = new BytesRef();
+ for (int i = 0; i < leftValues.getValueCount(); i++) {
+ leftValues.lookupOrd(i, scratchLeft);
+ rightValues.lookupOrd(i, scratchRight);
+ assertEquals(info, scratchLeft, scratchRight);
+ }
+ // bytes
+ for(int docID=0;docID<leftReader.maxDoc();docID++) {
+ leftValues.get(docID, scratchLeft);
+ rightValues.get(docID, scratchRight);
+ assertEquals(info, scratchLeft, scratchRight);
+ }
+ } else {
+ assertNull(info, leftValues);
+ assertNull(info, rightValues);
+ }
+ }
+
+ {
+ SortedSetDocValues leftValues = MultiDocValues.getSortedSetValues(leftReader, field);
+ SortedSetDocValues rightValues = MultiDocValues.getSortedSetValues(rightReader, field);
+ if (leftValues != null && rightValues != null) {
+ // numOrds
+ assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
+ // ords
+ BytesRef scratchLeft = new BytesRef();
+ BytesRef scratchRight = new BytesRef();
+ for (int i = 0; i < leftValues.getValueCount(); i++) {
+ leftValues.lookupOrd(i, scratchLeft);
+ rightValues.lookupOrd(i, scratchRight);
+ assertEquals(info, scratchLeft, scratchRight);
+ }
+ // ord lists
+ for(int docID=0;docID<leftReader.maxDoc();docID++) {
+ leftValues.setDocument(docID);
+ rightValues.setDocument(docID);
+ long ord;
+ while ((ord = leftValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ assertEquals(info, ord, rightValues.nextOrd());
+ }
+ assertEquals(info, SortedSetDocValues.NO_MORE_ORDS, rightValues.nextOrd());
+ }
+ } else {
+ assertNull(info, leftValues);
+ assertNull(info, rightValues);
+ }
}
}
}
- public void assertDocValues(DocValues leftDocValues, DocValues rightDocValues) throws Exception {
+ public void assertDocValues(int num, NumericDocValues leftDocValues, NumericDocValues rightDocValues) throws Exception {
assertNotNull(info, leftDocValues);
assertNotNull(info, rightDocValues);
- assertEquals(info, leftDocValues.getType(), rightDocValues.getType());
- assertEquals(info, leftDocValues.getValueSize(), rightDocValues.getValueSize());
- assertDocValuesSource(leftDocValues.getDirectSource(), rightDocValues.getDirectSource());
- assertDocValuesSource(leftDocValues.getSource(), rightDocValues.getSource());
- }
-
- /**
- * checks source API
- */
- public void assertDocValuesSource(DocValues.Source left, DocValues.Source right) throws Exception {
- DocValues.Type leftType = left.getType();
- assertEquals(info, leftType, right.getType());
- switch(leftType) {
- case VAR_INTS:
- case FIXED_INTS_8:
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- for (int i = 0; i < leftReader.maxDoc(); i++) {
- assertEquals(info, left.getInt(i), right.getInt(i));
- }
- break;
- case FLOAT_32:
- case FLOAT_64:
- for (int i = 0; i < leftReader.maxDoc(); i++) {
- assertEquals(info, left.getFloat(i), right.getFloat(i), 0F);
- }
- break;
- case BYTES_FIXED_STRAIGHT:
- case BYTES_FIXED_DEREF:
- case BYTES_VAR_STRAIGHT:
- case BYTES_VAR_DEREF:
- BytesRef b1 = new BytesRef();
- BytesRef b2 = new BytesRef();
- for (int i = 0; i < leftReader.maxDoc(); i++) {
- left.getBytes(i, b1);
- right.getBytes(i, b2);
- assertEquals(info, b1, b2);
- }
- break;
- // TODO: can we test these?
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
+ for(int docID=0;docID<num;docID++) {
+ assertEquals(leftDocValues.get(docID),
+ rightDocValues.get(docID));
}
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java Sun Mar 10 15:04:57 2013
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.Locale;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -71,7 +72,7 @@ public class TestFieldReplacements exten
final String[] allTokens = loremIpsum.split("\\s");
for (int index = random().nextInt(2 + i); index < allTokens.length; index += 1 + random()
.nextInt(2 + i)) {
- tokens.add(allTokens[index].toLowerCase());
+ tokens.add(allTokens[index].toLowerCase(Locale.ROOT));
}
fieldTokens[i] = tokens.toArray(new String[tokens.size()]);
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdates.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdates.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdates.java Sun Mar 10 15:04:57 2013
@@ -117,7 +117,7 @@ public class TestFieldUpdates extends Lu
// omitNorms is true
for (FieldInfo fi : reader.getFieldInfos()) {
if (fi.isIndexed()) {
- assertTrue(fi.omitsNorms() == (reader.normValues(fi.name) == null));
+ assertTrue("omitNorms=" + fi.omitsNorms() + " normsValue=" + reader.getNormValues(fi.name), fi.omitsNorms() == (reader.getNormValues(fi.name) == null));
}
}
reader.close();
Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java Sun Mar 10 15:04:57 2013
@@ -48,11 +48,12 @@ import org.junit.BeforeClass;
public class TestFieldsReader extends LuceneTestCase {
private static Directory dir;
- private static Document testDoc = new Document();
+ private static Document testDoc;
private static FieldInfos.Builder fieldInfos = null;
@BeforeClass
public static void beforeClass() throws Exception {
+ testDoc = new Document();
fieldInfos = new FieldInfos.Builder();
DocHelper.setupDoc(testDoc);
for (IndexableField field : testDoc.getFields()) {
@@ -291,12 +292,12 @@ public class TestFieldsReader extends Lu
for(AtomicReaderContext ctx : r.leaves()) {
final AtomicReader sub = ctx.reader();
- final int[] ids = FieldCache.DEFAULT.getInts(sub, "id", false);
+ final FieldCache.Ints ids = FieldCache.DEFAULT.getInts(sub, "id", false);
for(int docID=0;docID<sub.numDocs();docID++) {
final StoredDocument doc = sub.document(docID);
final Field f = (Field) doc.getField("nf");
assertTrue("got f=" + f, f instanceof StoredField);
- assertEquals(answers[ids[docID]], f.numericValue());
+ assertEquals(answers[ids.get(docID)], f.numericValue());
}
}
r.close();