You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/01/29 00:53:02 UTC
svn commit: r1439699 - in
/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index:
TestDocValuesIndexing.java TestDocValuesWithThreads.java
Author: rmuir
Date: Mon Jan 28 23:53:02 2013
New Revision: 1439699
URL: http://svn.apache.org/viewvc?rev=1439699&view=rev
Log:
nuke old test, move threaded test to WithThreads
Modified:
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1439699&r1=1439698&r2=1439699&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Mon Jan 28 23:53:02 2013
@@ -16,19 +16,8 @@ package org.apache.lucene.index;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import java.io.Closeable;
+
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -39,28 +28,15 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
-import org.junit.Assume;
/**
*
- * Tests DocValues integration into IndexWriter & Codecs
+ * Tests DocValues integration into IndexWriter
*
*/
public class TestDocValuesIndexing extends LuceneTestCase {
@@ -69,22 +45,6 @@ public class TestDocValuesIndexing exten
* - add multithreaded tests / integrate into stress indexing?
*/
- public void testIndexBytesNoDeletes() throws IOException {
- runTestIndexBytes(writerConfig(random().nextBoolean()), false);
- }
-
- public void testIndexBytesDeletes() throws IOException {
- runTestIndexBytes(writerConfig(random().nextBoolean()), true);
- }
-
- public void testIndexNumericsNoDeletes() throws IOException {
- runTestNumerics(writerConfig(random().nextBoolean()), false);
- }
-
- public void testIndexNumericsDeletes() throws IOException {
- runTestNumerics(writerConfig(random().nextBoolean()), true);
- }
-
public void testAddIndexes() throws IOException {
Directory d1 = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d1);
@@ -123,571 +83,6 @@ public class TestDocValuesIndexing exten
d3.close();
}
- public void testAddIndexesRandom() throws IOException {
- //nocommit convert
- /*
- int valuesPerIndex = 10;
- List<Type> values = Arrays.asList(Type.values());
- Collections.shuffle(values, random());
- Type first = values.get(0);
- Type second = values.get(1);
- // index first index
- Directory d_1 = newDirectory();
- IndexWriter w_1 = new IndexWriter(d_1, writerConfig(random().nextBoolean()));
- indexValues(w_1, valuesPerIndex, first, values, false, 7);
- w_1.commit();
- assertEquals(valuesPerIndex, w_1.maxDoc());
- _TestUtil.checkIndex(d_1);
-
- // index second index
- Directory d_2 = newDirectory();
- IndexWriter w_2 = new IndexWriter(d_2, writerConfig(random().nextBoolean()));
- indexValues(w_2, valuesPerIndex, second, values, false, 7);
- w_2.commit();
- assertEquals(valuesPerIndex, w_2.maxDoc());
- _TestUtil.checkIndex(d_2);
-
- Directory target = newDirectory();
- IndexWriter w = new IndexWriter(target, writerConfig(random().nextBoolean()));
- DirectoryReader r_1 = DirectoryReader.open(w_1, true);
- DirectoryReader r_2 = DirectoryReader.open(w_2, true);
- if (random().nextBoolean()) {
- w.addIndexes(d_1, d_2);
- } else {
- w.addIndexes(r_1, r_2);
- }
- w.forceMerge(1, true);
- w.commit();
-
- _TestUtil.checkIndex(target);
- assertEquals(valuesPerIndex * 2, w.maxDoc());
-
- // check values
-
- DirectoryReader merged = DirectoryReader.open(w, true);
- Source source_1 = getSource(getDocValues(r_1, first.name()));
- Source source_2 = getSource(getDocValues(r_2, second.name()));
- Source source_1_merged = getSource(getDocValues(merged, first.name()));
- Source source_2_merged = getSource(getDocValues(merged, second
- .name()));
- for (int i = 0; i < r_1.maxDoc(); i++) {
- switch (first) {
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_SORTED:
- assertEquals(source_1.getBytes(i, new BytesRef()),
- source_1_merged.getBytes(i, new BytesRef()));
- break;
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- case VAR_INTS:
- assertEquals(source_1.getInt(i), source_1_merged.getInt(i));
- break;
- case FLOAT_32:
- case FLOAT_64:
- assertEquals(source_1.getFloat(i), source_1_merged.getFloat(i), 0.0d);
- break;
- default:
- fail("unkonwn " + first);
- }
- }
-
- for (int i = r_1.maxDoc(); i < merged.maxDoc(); i++) {
- switch (second) {
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_SORTED:
- assertEquals(source_2.getBytes(i - r_1.maxDoc(), new BytesRef()),
- source_2_merged.getBytes(i, new BytesRef()));
- break;
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- case VAR_INTS:
- assertEquals(source_2.getInt(i - r_1.maxDoc()),
- source_2_merged.getInt(i));
- break;
- case FLOAT_32:
- case FLOAT_64:
- assertEquals(source_2.getFloat(i - r_1.maxDoc()),
- source_2_merged.getFloat(i), 0.0d);
- break;
- default:
- fail("unkonwn " + first);
- }
- }
- // close resources
- r_1.close();
- r_2.close();
- merged.close();
- w_1.close(true);
- w_2.close(true);
- w.close(true);
- d_1.close();
- d_2.close();
- target.close();
- */
- }
-
- private IndexWriterConfig writerConfig(boolean useCompoundFile) {
- final IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random()));
- cfg.setMergePolicy(newLogMergePolicy(random()));
- LogMergePolicy policy = new LogDocMergePolicy();
- cfg.setMergePolicy(policy);
- policy.setUseCompoundFile(useCompoundFile);
- return cfg;
- }
-
- @SuppressWarnings("fallthrough")
- public void runTestNumerics(IndexWriterConfig cfg, boolean withDeletions)
- throws IOException {
- //nocommit convert
- /*
- Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, cfg);
- final int numValues = 50 + atLeast(10);
- final List<Type> numVariantList = new ArrayList<Type>(NUMERICS);
-
- // run in random order to test if fill works correctly during merges
- Collections.shuffle(numVariantList, random());
- for (Type val : numVariantList) {
- FixedBitSet deleted = indexValues(w, numValues, val, numVariantList,
- withDeletions, 7);
- List<Closeable> closeables = new ArrayList<Closeable>();
- DirectoryReader r = DirectoryReader.open(w, true);
- final int numRemainingValues = numValues - deleted.cardinality();
- final int base = r.numDocs() - numRemainingValues;
- // for FIXED_INTS_8 we use value mod 128 - to enable testing in
- // one go we simply use numValues as the mod for all other INT types
- int mod = numValues;
- switch (val) {
- case FIXED_INTS_8:
- mod = 128;
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case VAR_INTS: {
- DocValues intsReader = getDocValues(r, val.name());
- assertNotNull(intsReader);
-
- Source ints = getSource(intsReader);
-
- for (int i = 0; i < base; i++) {
- long value = ints.getInt(i);
- assertEquals("index " + i, 0, value);
- }
-
- int expected = 0;
- for (int i = base; i < r.numDocs(); i++, expected++) {
- while (deleted.get(expected)) {
- expected++;
- }
- assertEquals(val + " mod: " + mod + " index: " + i, expected%mod, ints.getInt(i));
- }
- }
- break;
- case FLOAT_32:
- case FLOAT_64: {
- DocValues floatReader = getDocValues(r, val.name());
- assertNotNull(floatReader);
- Source floats = getSource(floatReader);
- for (int i = 0; i < base; i++) {
- double value = floats.getFloat(i);
- assertEquals(val + " failed for doc: " + i + " base: " + base,
- 0.0d, value, 0.0d);
- }
- int expected = 0;
- for (int i = base; i < r.numDocs(); i++, expected++) {
- while (deleted.get(expected)) {
- expected++;
- }
- assertEquals("index " + i, 2.0 * expected, floats.getFloat(i),
- 0.00001);
- }
- }
- break;
- default:
- fail("unexpected value " + val);
- }
-
- closeables.add(r);
- for (Closeable toClose : closeables) {
- toClose.close();
- }
- }
- w.close();
- d.close();
- */
- }
-
- public void runTestIndexBytes(IndexWriterConfig cfg, boolean withDeletions)
- throws IOException {
- /* nocommit convert
- final Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, cfg);
- final List<Type> byteVariantList = new ArrayList<Type>(BYTES);
- // run in random order to test if fill works correctly during merges
- Collections.shuffle(byteVariantList, random());
- final int numValues = 50 + atLeast(10);
- for (Type byteIndexValue : byteVariantList) {
- List<Closeable> closeables = new ArrayList<Closeable>();
- final int bytesSize = 1 + atLeast(50);
- FixedBitSet deleted = indexValues(w, numValues, byteIndexValue,
- byteVariantList, withDeletions, bytesSize);
- final DirectoryReader r = DirectoryReader.open(w, withDeletions);
- assertEquals(0, r.numDeletedDocs());
- final int numRemainingValues = numValues - deleted.cardinality();
- final int base = r.numDocs() - numRemainingValues;
- DocValues bytesReader = getDocValues(r, byteIndexValue.name());
- assertNotNull("field " + byteIndexValue.name()
- + " returned null reader - maybe merged failed", bytesReader);
- Source bytes = getSource(bytesReader);
- byte upto = 0;
-
- // test the filled up slots for correctness
- for (int i = 0; i < base; i++) {
-
- BytesRef br = bytes.getBytes(i, new BytesRef());
- String msg = " field: " + byteIndexValue.name() + " at index: " + i
- + " base: " + base + " numDocs:" + r.numDocs();
- switch (byteIndexValue) {
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
- // fixed straight returns bytesref with zero bytes all of fixed
- // length
- assertNotNull("expected none null - " + msg, br);
- if (br.length != 0) {
- assertEquals("expected zero bytes of length " + bytesSize + " - "
- + msg + br.utf8ToString(), bytesSize, br.length);
- for (int j = 0; j < br.length; j++) {
- assertEquals("Byte at index " + j + " doesn't match - " + msg, 0,
- br.bytes[br.offset + j]);
- }
- }
- break;
- default:
- assertNotNull("expected none null - " + msg, br);
- assertEquals(byteIndexValue + "", 0, br.length);
- // make sure we advance at least until base
- }
- }
-
- // test the actual doc values added in this iteration
- assertEquals(base + numRemainingValues, r.numDocs());
- int v = 0;
- for (int i = base; i < r.numDocs(); i++) {
- String msg = " field: " + byteIndexValue.name() + " at index: " + i
- + " base: " + base + " numDocs:" + r.numDocs() + " bytesSize: "
- + bytesSize + " src: " + bytes;
- while (withDeletions && deleted.get(v++)) {
- upto += bytesSize;
- }
- BytesRef br = bytes.getBytes(i, new BytesRef());
- assertTrue(msg, br.length > 0);
- for (int j = 0; j < br.length; j++, upto++) {
- if (!(br.bytes.length > br.offset + j))
- br = bytes.getBytes(i, new BytesRef());
- assertTrue("BytesRef index exceeded [" + msg + "] offset: "
- + br.offset + " length: " + br.length + " index: "
- + (br.offset + j), br.bytes.length > br.offset + j);
- assertEquals("SourceRef Byte at index " + j + " doesn't match - "
- + msg, upto, br.bytes[br.offset + j]);
- }
- }
-
- // clean up
- closeables.add(r);
- for (Closeable toClose : closeables) {
- toClose.close();
- }
- }
-
- w.close();
- d.close();
- */
- }
-
- public void testGetArrayNumerics() throws IOException {
- /* nocommit conver
- Directory d = newDirectory();
- IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter w = new IndexWriter(d, cfg);
- final int numValues = 50 + atLeast(10);
- final List<Type> numVariantList = new ArrayList<Type>(NUMERICS);
- Collections.shuffle(numVariantList, random());
- for (Type val : numVariantList) {
- indexValues(w, numValues, val, numVariantList,
- false, 7);
- DirectoryReader r = DirectoryReader.open(w, true);
- DocValues docValues = getDocValues(r, val.name());
- assertNotNull(docValues);
- // make sure we don't get a direct source since they don't support getArray()
- Source source = docValues.getSource();
- switch (source.getType()) {
- case FIXED_INTS_8:
- {
- assertTrue(source.hasArray());
- byte[] values = (byte[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals((long)values[i], source.getInt(i));
- }
- }
- break;
- case FIXED_INTS_16:
- {
- assertTrue(source.hasArray());
- short[] values = (short[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals((long)values[i], source.getInt(i));
- }
- }
- break;
- case FIXED_INTS_32:
- {
- assertTrue(source.hasArray());
- int[] values = (int[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals((long)values[i], source.getInt(i));
- }
- }
- break;
- case FIXED_INTS_64:
- {
- assertTrue(source.hasArray());
- long[] values = (long[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals(values[i], source.getInt(i));
- }
- }
- break;
- case VAR_INTS:
- assertFalse(source.hasArray());
- break;
- case FLOAT_32:
- {
- assertTrue(source.hasArray());
- float[] values = (float[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals((double)values[i], source.getFloat(i), 0.0d);
- }
- }
- break;
- case FLOAT_64:
- {
- assertTrue(source.hasArray());
- double[] values = (double[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- assertEquals(values[i], source.getFloat(i), 0.0d);
- }
- }
- break;
- default:
- fail("unexpected value " + source.getType());
- }
- r.close();
- }
- w.close();
- d.close();
- */
- }
-
- public void testGetArrayBytes() throws IOException {
- /* nocommit convert
- Directory d = newDirectory();
- IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random()));
- IndexWriter w = new IndexWriter(d, cfg);
- final int numValues = 50 + atLeast(10);
- // only single byte fixed straight supports getArray()
- indexValues(w, numValues, Type.BYTES_FIXED_STRAIGHT, null, false, 1);
- DirectoryReader r = DirectoryReader.open(w, true);
- DocValues docValues = getDocValues(r, Type.BYTES_FIXED_STRAIGHT.name());
- assertNotNull(docValues);
- // make sure we don't get a direct source since they don't support
- // getArray()
- Source source = docValues.getSource();
-
- switch (source.getType()) {
- case BYTES_FIXED_STRAIGHT: {
- BytesRef ref = new BytesRef();
- if (source.hasArray()) {
- byte[] values = (byte[]) source.getArray();
- for (int i = 0; i < numValues; i++) {
- source.getBytes(i, ref);
- assertEquals(1, ref.length);
- assertEquals(values[i], ref.bytes[ref.offset]);
- }
- }
- }
- break;
- default:
- fail("unexpected value " + source.getType());
- }
- r.close();
- w.close();
- d.close();
- */
- }
-
- /* Nocommit convert all this
-
- private static EnumSet<Type> BYTES = EnumSet.of(Type.BYTES_FIXED_DEREF,
- Type.BYTES_FIXED_STRAIGHT, Type.BYTES_VAR_DEREF,
- Type.BYTES_VAR_STRAIGHT, Type.BYTES_FIXED_SORTED, Type.BYTES_VAR_SORTED);
-
- private static EnumSet<Type> NUMERICS = EnumSet.of(Type.VAR_INTS,
- Type.FIXED_INTS_16, Type.FIXED_INTS_32,
- Type.FIXED_INTS_64,
- Type.FIXED_INTS_8,
- Type.FLOAT_32,
- Type.FLOAT_64);
-
- private FixedBitSet indexValues(IndexWriter w, int numValues, Type valueType,
- List<Type> valueVarList, boolean withDeletions, int bytesSize)
- throws IOException {
- final boolean isNumeric = NUMERICS.contains(valueType);
- FixedBitSet deleted = new FixedBitSet(numValues);
- Document doc = new Document();
- final Field valField;
- if (isNumeric) {
- switch (valueType) {
- case VAR_INTS:
- valField = new PackedLongDocValuesField(valueType.name(), (long) 0);
- break;
- case FIXED_INTS_16:
- valField = new ShortDocValuesField(valueType.name(), (short) 0);
- break;
- case FIXED_INTS_32:
- valField = new IntDocValuesField(valueType.name(), 0);
- break;
- case FIXED_INTS_64:
- valField = new LongDocValuesField(valueType.name(), (long) 0);
- break;
- case FIXED_INTS_8:
- valField = new ByteDocValuesField(valueType.name(), (byte) 0);
- break;
- case FLOAT_32:
- valField = new FloatDocValuesField(valueType.name(), (float) 0);
- break;
- case FLOAT_64:
- valField = new DoubleDocValuesField(valueType.name(), (double) 0);
- break;
- default:
- valField = null;
- fail("unhandled case");
- }
- } else {
- switch (valueType) {
- case BYTES_FIXED_STRAIGHT:
- valField = new StraightBytesDocValuesField(valueType.name(), new BytesRef(), true);
- break;
- case BYTES_VAR_STRAIGHT:
- valField = new StraightBytesDocValuesField(valueType.name(), new BytesRef(), false);
- break;
- case BYTES_FIXED_DEREF:
- valField = new DerefBytesDocValuesField(valueType.name(), new BytesRef(), true);
- break;
- case BYTES_VAR_DEREF:
- valField = new DerefBytesDocValuesField(valueType.name(), new BytesRef(), false);
- break;
- case BYTES_FIXED_SORTED:
- valField = new SortedBytesDocValuesField(valueType.name(), new BytesRef(), true);
- break;
- case BYTES_VAR_SORTED:
- valField = new SortedBytesDocValuesField(valueType.name(), new BytesRef(), false);
- break;
- default:
- valField = null;
- fail("unhandled case");
- }
- }
- doc.add(valField);
- final BytesRef bytesRef = new BytesRef();
-
- final String idBase = valueType.name() + "_";
- final byte[] b = new byte[bytesSize];
- if (bytesRef != null) {
- bytesRef.bytes = b;
- bytesRef.length = b.length;
- bytesRef.offset = 0;
- }
- byte upto = 0;
- for (int i = 0; i < numValues; i++) {
- if (isNumeric) {
- switch (valueType) {
- case VAR_INTS:
- valField.setLongValue((long)i);
- break;
- case FIXED_INTS_16:
- valField.setShortValue((short)i);
- break;
- case FIXED_INTS_32:
- valField.setIntValue(i);
- break;
- case FIXED_INTS_64:
- valField.setLongValue((long)i);
- break;
- case FIXED_INTS_8:
- valField.setByteValue((byte)(0xFF & (i % 128)));
- break;
- case FLOAT_32:
- valField.setFloatValue(2.0f * i);
- break;
- case FLOAT_64:
- valField.setDoubleValue(2.0d * i);
- break;
- default:
- fail("unexpected value " + valueType);
- }
- } else {
- for (int j = 0; j < b.length; j++) {
- b[j] = upto++;
- }
- if (bytesRef != null) {
- valField.setBytesValue(bytesRef);
- }
- }
- doc.removeFields("id");
- doc.add(new StringField("id", idBase + i, Field.Store.YES));
- w.addDocument(doc);
-
- if (i % 7 == 0) {
- if (withDeletions && random().nextBoolean()) {
- Type val = valueVarList.get(random().nextInt(1 + valueVarList
- .indexOf(valueType)));
- final int randInt = val == valueType ? random().nextInt(1 + i) : random()
- .nextInt(numValues);
- w.deleteDocuments(new Term("id", val.name() + "_" + randInt));
- if (val == valueType) {
- deleted.set(randInt);
- }
- }
- if (random().nextInt(10) == 0) {
- w.commit();
- }
- }
- }
- w.commit();
-
- // TODO test multi seg with deletions
- if (withDeletions || random().nextBoolean()) {
- w.forceMerge(1, true);
- }
- return deleted;
- }*/
-
public void testMultiValuedDocValuesField() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
@@ -768,101 +163,6 @@ public class TestDocValuesIndexing exten
d.close();
}
- public void testWithThreads() throws Exception {
- Random random = random();
- final int NUM_DOCS = atLeast(100);
- final Directory dir = newDirectory();
- final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
- final boolean allowDups = random.nextBoolean();
- final Set<String> seen = new HashSet<String>();
- if (VERBOSE) {
- System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " allowDups=" + allowDups);
- }
- int numDocs = 0;
- final List<BytesRef> docValues = new ArrayList<BytesRef>();
-
- // TODO: deletions
- while (numDocs < NUM_DOCS) {
- final String s;
- if (random.nextBoolean()) {
- s = _TestUtil.randomSimpleString(random);
- } else {
- s = _TestUtil.randomUnicodeString(random);
- }
- final BytesRef br = new BytesRef(s);
-
- if (!allowDups) {
- if (seen.contains(s)) {
- continue;
- }
- seen.add(s);
- }
-
- if (VERBOSE) {
- System.out.println(" " + numDocs + ": s=" + s);
- }
-
- final Document doc = new Document();
- doc.add(new SortedDocValuesField("stringdv", br));
- doc.add(new NumericDocValuesField("id", numDocs));
- docValues.add(br);
- writer.addDocument(doc);
- numDocs++;
-
- if (random.nextInt(40) == 17) {
- // force flush
- writer.getReader().close();
- }
- }
-
- writer.forceMerge(1);
- final DirectoryReader r = writer.getReader();
- writer.close();
-
- final AtomicReader sr = getOnlySegmentReader(r);
-
- final long END_TIME = System.currentTimeMillis() + (TEST_NIGHTLY ? 30 : 1);
-
- final NumericDocValues docIDToID = numeric(sr, "id");
-
- final int NUM_THREADS = _TestUtil.nextInt(random(), 1, 10);
- Thread[] threads = new Thread[NUM_THREADS];
- for(int thread=0;thread<NUM_THREADS;thread++) {
- threads[thread] = new Thread() {
- @Override
- public void run() {
- Random random = random();
- final SortedDocValues stringDVDirect;
- try {
- stringDVDirect = sr.getSortedDocValues("stringdv");
- assertNotNull(stringDVDirect);
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
- }
- while(System.currentTimeMillis() < END_TIME) {
- final SortedDocValues source;
- source = stringDVDirect;
- final BytesRef scratch = new BytesRef();
-
- for(int iter=0;iter<100;iter++) {
- final int docID = random.nextInt(sr.maxDoc());
- source.get(docID, scratch);
- assertEquals(docValues.get((int) docIDToID.get(docID)), scratch);
- }
- }
- }
- };
- threads[thread].start();
- }
-
- for(Thread thread : threads) {
- thread.join();
- }
-
- r.close();
- dir.close();
- }
-
// LUCENE-3870
public void testLengthPrefixAcrossTwoPages() throws Exception {
Directory d = newDirectory();
@@ -1209,37 +509,4 @@ public class TestDocValuesIndexing exten
w.close();
dir.close();
}
-
- public NumericDocValues numeric(AtomicReader reader, String field) throws IOException {
- NumericDocValues docValues = reader.getNumericDocValues(field);
- if(random().nextBoolean()) {
- return docValues;
- }
- return docValues;
- }
-
- public NumericDocValues numeric(DirectoryReader reader, String field) throws IOException {
- return numeric(getOnlySegmentReader(reader), field);
- }
- public BinaryDocValues binary(DirectoryReader reader, String field) throws IOException {
- return binary(getOnlySegmentReader(reader), field);
- }
- public SortedDocValues sorted(DirectoryReader reader, String field) throws IOException {
- return sorted(getOnlySegmentReader(reader), field);
- }
-
- public BinaryDocValues binary(AtomicReader reader, String field) throws IOException {
- BinaryDocValues docValues = reader.getBinaryDocValues(field);
- if(random().nextBoolean()) {
- return docValues;
- }
- return docValues;
- }
- public SortedDocValues sorted(AtomicReader reader, String field) throws IOException {
- SortedDocValues docValues = reader.getSortedDocValues(field);
- if(random().nextBoolean()) {
- return docValues;
- }
- return docValues;
- }
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java?rev=1439699&r1=1439698&r2=1439699&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java Mon Jan 28 23:53:02 2013
@@ -17,9 +17,12 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Random;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -128,5 +131,100 @@ public class TestDocValuesWithThreads ex
r.close();
dir.close();
}
+
+ public void test2() throws Exception {
+ Random random = random();
+ final int NUM_DOCS = atLeast(100);
+ final Directory dir = newDirectory();
+ final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
+ final boolean allowDups = random.nextBoolean();
+ final Set<String> seen = new HashSet<String>();
+ if (VERBOSE) {
+ System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " allowDups=" + allowDups);
+ }
+ int numDocs = 0;
+ final List<BytesRef> docValues = new ArrayList<BytesRef>();
+
+ // TODO: deletions
+ while (numDocs < NUM_DOCS) {
+ final String s;
+ if (random.nextBoolean()) {
+ s = _TestUtil.randomSimpleString(random);
+ } else {
+ s = _TestUtil.randomUnicodeString(random);
+ }
+ final BytesRef br = new BytesRef(s);
+
+ if (!allowDups) {
+ if (seen.contains(s)) {
+ continue;
+ }
+ seen.add(s);
+ }
+
+ if (VERBOSE) {
+ System.out.println(" " + numDocs + ": s=" + s);
+ }
+
+ final Document doc = new Document();
+ doc.add(new SortedDocValuesField("stringdv", br));
+ doc.add(new NumericDocValuesField("id", numDocs));
+ docValues.add(br);
+ writer.addDocument(doc);
+ numDocs++;
+
+ if (random.nextInt(40) == 17) {
+ // force flush
+ writer.getReader().close();
+ }
+ }
+
+ writer.forceMerge(1);
+ final DirectoryReader r = writer.getReader();
+ writer.close();
+
+ final AtomicReader sr = getOnlySegmentReader(r);
+
+ final long END_TIME = System.currentTimeMillis() + (TEST_NIGHTLY ? 30 : 1);
+
+ final NumericDocValues docIDToID = sr.getNumericDocValues("id");
+
+ final int NUM_THREADS = _TestUtil.nextInt(random(), 1, 10);
+ Thread[] threads = new Thread[NUM_THREADS];
+ for(int thread=0;thread<NUM_THREADS;thread++) {
+ threads[thread] = new Thread() {
+ @Override
+ public void run() {
+ Random random = random();
+ final SortedDocValues stringDVDirect;
+ try {
+ stringDVDirect = sr.getSortedDocValues("stringdv");
+ assertNotNull(stringDVDirect);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ while(System.currentTimeMillis() < END_TIME) {
+ final SortedDocValues source;
+ source = stringDVDirect;
+ final BytesRef scratch = new BytesRef();
+
+ for(int iter=0;iter<100;iter++) {
+ final int docID = random.nextInt(sr.maxDoc());
+ source.get(docID, scratch);
+ assertEquals(docValues.get((int) docIDToID.get(docID)), scratch);
+ }
+ }
+ }
+ };
+ threads[thread].start();
+ }
+
+ for(Thread thread : threads) {
+ thread.join();
+ }
+
+ r.close();
+ dir.close();
+ }
}