You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/08/23 16:07:19 UTC
svn commit: r1160700 [9/22] - in /lucene/dev/branches/flexscoring: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/lucene/contrib/demo/
dev-tools/idea/lucene/contrib/highlighter/ dev-tools/idea/lucene/contrib/q...
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java Tue Aug 23 14:06:58 2011
@@ -30,8 +30,8 @@ import org.apache.lucene.index.codecs.Fi
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
import org.apache.lucene.index.codecs.sep.IntIndexInput;
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
-import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
-import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
+import org.apache.lucene.index.codecs.sep.SepPostingsReader;
+import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
@@ -127,7 +127,7 @@ public class MockFixedIntBlockCodec exte
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new SepPostingsWriterImpl(state, new MockIntFactory(blockSize));
+ PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockIntFactory(blockSize));
boolean success = false;
TermsIndexWriterBase indexWriter;
@@ -158,10 +158,10 @@ public class MockFixedIntBlockCodec exte
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
- state.segmentInfo,
- state.context,
- new MockIntFactory(blockSize), state.codecId);
+ PostingsReaderBase postingsReader = new SepPostingsReader(state.dir,
+ state.segmentInfo,
+ state.context,
+ new MockIntFactory(blockSize), state.codecId);
TermsIndexReaderBase indexReader;
boolean success = false;
@@ -204,7 +204,7 @@ public class MockFixedIntBlockCodec exte
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
- SepPostingsReaderImpl.files(segmentInfo, codecId, files);
+ SepPostingsReader.files(segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
@@ -212,7 +212,7 @@ public class MockFixedIntBlockCodec exte
@Override
public void getExtensions(Set<String> extensions) {
- SepPostingsWriterImpl.getExtensions(extensions);
+ SepPostingsWriter.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java Tue Aug 23 14:06:58 2011
@@ -30,8 +30,8 @@ import org.apache.lucene.index.codecs.Fi
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
import org.apache.lucene.index.codecs.sep.IntIndexInput;
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
-import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
-import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
+import org.apache.lucene.index.codecs.sep.SepPostingsReader;
+import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput;
import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput;
@@ -150,7 +150,7 @@ public class MockVariableIntBlockCodec e
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new SepPostingsWriterImpl(state, new MockIntFactory(baseBlockSize));
+ PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockIntFactory(baseBlockSize));
boolean success = false;
TermsIndexWriterBase indexWriter;
@@ -181,10 +181,10 @@ public class MockVariableIntBlockCodec e
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir,
- state.segmentInfo,
- state.context,
- new MockIntFactory(baseBlockSize), state.codecId);
+ PostingsReaderBase postingsReader = new SepPostingsReader(state.dir,
+ state.segmentInfo,
+ state.context,
+ new MockIntFactory(baseBlockSize), state.codecId);
TermsIndexReaderBase indexReader;
boolean success = false;
@@ -227,7 +227,7 @@ public class MockVariableIntBlockCodec e
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
- SepPostingsReaderImpl.files(segmentInfo, codecId, files);
+ SepPostingsReader.files(segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
@@ -235,7 +235,7 @@ public class MockVariableIntBlockCodec e
@Override
public void getExtensions(Set<String> extensions) {
- SepPostingsWriterImpl.getExtensions(extensions);
+ SepPostingsWriter.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java Tue Aug 23 14:06:58 2011
@@ -30,6 +30,8 @@ import org.apache.lucene.index.PerDocWri
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.codecs.BlockTreeTermsReader;
+import org.apache.lucene.index.codecs.BlockTreeTermsWriter;
import org.apache.lucene.index.codecs.BlockTermsReader;
import org.apache.lucene.index.codecs.BlockTermsWriter;
import org.apache.lucene.index.codecs.Codec;
@@ -51,13 +53,13 @@ import org.apache.lucene.index.codecs.Va
import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec;
import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec;
import org.apache.lucene.index.codecs.mocksep.MockSingleIntFactory;
-import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReaderImpl;
-import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl;
+import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReader;
+import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter;
import org.apache.lucene.index.codecs.sep.IntIndexInput;
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
-import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
-import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
+import org.apache.lucene.index.codecs.sep.SepPostingsReader;
+import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
import org.apache.lucene.index.codecs.standard.StandardPostingsWriter;
import org.apache.lucene.store.Directory;
@@ -137,7 +139,7 @@ public class MockRandomCodec extends Cod
final long seed = seedRandom.nextLong();
if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " seed=" + seed);
+ System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " codecID=" + state.codecId + " seed=" + seed);
}
final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SEED_EXT);
@@ -151,11 +153,10 @@ public class MockRandomCodec extends Cod
final Random random = new Random(seed);
random.nextInt(); // consume a random for buffersize
-
- PostingsWriterBase postingsWriter;
+ PostingsWriterBase postingsWriter;
if (random.nextBoolean()) {
- postingsWriter = new SepPostingsWriterImpl(state, new MockIntStreamFactory(random), skipInterval);
+ postingsWriter = new SepPostingsWriter(state, new MockIntStreamFactory(random), skipInterval);
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing Standard postings");
@@ -166,76 +167,107 @@ public class MockRandomCodec extends Cod
if (random.nextBoolean()) {
final int totTFCutoff = _TestUtil.nextInt(random, 1, 20);
if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: pulsing postings with totTFCutoff=" + totTFCutoff);
+ System.out.println("MockRandomCodec: writing pulsing postings with totTFCutoff=" + totTFCutoff);
}
- postingsWriter = new PulsingPostingsWriterImpl(totTFCutoff, postingsWriter);
+ postingsWriter = new PulsingPostingsWriter(totTFCutoff, postingsWriter);
}
- final TermsIndexWriterBase indexWriter;
- boolean success = false;
+ final FieldsConsumer fields;
- try {
- if (random.nextBoolean()) {
- state.termIndexInterval = _TestUtil.nextInt(random, 1, 100);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: fixed-gap terms index (tii=" + state.termIndexInterval + ")");
+ if (random.nextBoolean()) {
+ // Use BlockTree terms dict
+
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: writing BlockTree terms dict");
+ }
+
+ // TODO: would be nice to allow 1 but this is very
+ // slow to write
+ final int minTermsInBlock = _TestUtil.nextInt(random, 2, 100);
+ final int maxTermsInBlock = Math.max(2, (minTermsInBlock-1)*2 + random.nextInt(100));
+
+ boolean success = false;
+ try {
+ fields = new BlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock);
+ success = true;
+ } finally {
+ if (!success) {
+ postingsWriter.close();
}
- indexWriter = new FixedGapTermsIndexWriter(state);
- } else {
- final VariableGapTermsIndexWriter.IndexTermSelector selector;
- final int n2 = random.nextInt(3);
- if (n2 == 0) {
- final int tii = _TestUtil.nextInt(random, 1, 100);
- selector = new VariableGapTermsIndexWriter.EveryNTermSelector(tii);
+ }
+ } else {
+
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: writing Block terms dict");
+ }
+
+ boolean success = false;
+
+ final TermsIndexWriterBase indexWriter;
+ try {
+ if (random.nextBoolean()) {
+ state.termIndexInterval = _TestUtil.nextInt(random, 1, 100);
if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: variable-gap terms index (tii=" + tii + ")");
+ System.out.println("MockRandomCodec: fixed-gap terms index (tii=" + state.termIndexInterval + ")");
}
- } else if (n2 == 1) {
- final int docFreqThresh = _TestUtil.nextInt(random, 2, 100);
- final int tii = _TestUtil.nextInt(random, 1, 100);
- selector = new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThresh, tii);
+ indexWriter = new FixedGapTermsIndexWriter(state);
} else {
- final long seed2 = random.nextLong();
- final int gap = _TestUtil.nextInt(random, 2, 40);
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: random-gap terms index (max gap=" + gap + ")");
+ final VariableGapTermsIndexWriter.IndexTermSelector selector;
+ final int n2 = random.nextInt(3);
+ if (n2 == 0) {
+ final int tii = _TestUtil.nextInt(random, 1, 100);
+ selector = new VariableGapTermsIndexWriter.EveryNTermSelector(tii);
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: variable-gap terms index (tii=" + tii + ")");
+ }
+ } else if (n2 == 1) {
+ final int docFreqThresh = _TestUtil.nextInt(random, 2, 100);
+ final int tii = _TestUtil.nextInt(random, 1, 100);
+ selector = new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThresh, tii);
+ } else {
+ final long seed2 = random.nextLong();
+ final int gap = _TestUtil.nextInt(random, 2, 40);
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: random-gap terms index (max gap=" + gap + ")");
+ }
+ selector = new VariableGapTermsIndexWriter.IndexTermSelector() {
+ final Random rand = new Random(seed2);
+
+ @Override
+ public boolean isIndexTerm(BytesRef term, TermStats stats) {
+ return rand.nextInt(gap) == gap/2;
+ }
+
+ @Override
+ public void newField(FieldInfo fieldInfo) {
+ }
+ };
}
- selector = new VariableGapTermsIndexWriter.IndexTermSelector() {
- final Random rand = new Random(seed2);
-
- @Override
- public boolean isIndexTerm(BytesRef term, TermStats stats) {
- return rand.nextInt(gap) == gap/2;
- }
-
- @Override
- public void newField(FieldInfo fieldInfo) {
- }
- };
+ indexWriter = new VariableGapTermsIndexWriter(state, selector);
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ postingsWriter.close();
}
- indexWriter = new VariableGapTermsIndexWriter(state, selector);
- }
- success = true;
- } finally {
- if (!success) {
- postingsWriter.close();
}
- }
- success = false;
- try {
- FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, postingsWriter);
- success = true;
- return ret;
- } finally {
- if (!success) {
- try {
- postingsWriter.close();
- } finally {
- indexWriter.close();
+ success = false;
+ try {
+ fields = new BlockTermsWriter(indexWriter, state, postingsWriter);
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ postingsWriter.close();
+ } finally {
+ indexWriter.close();
+ }
}
}
}
+
+ return fields;
}
@Override
@@ -245,7 +277,7 @@ public class MockRandomCodec extends Cod
final IndexInput in = state.dir.openInput(seedFileName, state.context);
final long seed = in.readLong();
if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " seed=" + seed);
+ System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " codecID=" + state.codecId + " seed=" + seed);
}
in.close();
@@ -259,8 +291,11 @@ public class MockRandomCodec extends Cod
PostingsReaderBase postingsReader;
if (random.nextBoolean()) {
- postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
- state.context, new MockIntStreamFactory(random), state.codecId);
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: reading Sep postings");
+ }
+ postingsReader = new SepPostingsReader(state.dir, state.segmentInfo,
+ state.context, new MockIntStreamFactory(random), state.codecId);
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Standard postings");
@@ -273,86 +308,119 @@ public class MockRandomCodec extends Cod
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading pulsing postings with totTFCutoff=" + totTFCutoff);
}
- postingsReader = new PulsingPostingsReaderImpl(postingsReader);
+ postingsReader = new PulsingPostingsReader(postingsReader);
}
- final TermsIndexReaderBase indexReader;
- boolean success = false;
+ final FieldsProducer fields;
- try {
- if (random.nextBoolean()) {
- // if termsIndexDivisor is set to -1, we should not touch it. It means a
- // test explicitly instructed not to load the terms index.
+ if (random.nextBoolean()) {
+ // Use BlockTree terms dict
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: reading BlockTree terms dict");
+ }
+
+ boolean success = false;
+ try {
+ fields = new BlockTreeTermsReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ postingsReader,
+ state.context,
+ state.codecId,
+ state.termsIndexDivisor);
+ success = true;
+ } finally {
+ if (!success) {
+ postingsReader.close();
+ }
+ }
+ } else {
+
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: reading Block terms dict");
+ }
+ final TermsIndexReaderBase indexReader;
+ boolean success = false;
+ try {
+ final boolean doFixedGap = random.nextBoolean();
+
+ // randomness diverges from writer, here:
if (state.termsIndexDivisor != -1) {
state.termsIndexDivisor = _TestUtil.nextInt(random, 1, 10);
}
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: fixed-gap terms index (divisor=" + state.termsIndexDivisor + ")");
- }
- indexReader = new FixedGapTermsIndexReader(state.dir,
- state.fieldInfos,
- state.segmentInfo.name,
- state.termsIndexDivisor,
- BytesRef.getUTF8SortedAsUnicodeComparator(),
- state.codecId, state.context);
- } else {
- final int n2 = random.nextInt(3);
- if (n2 == 1) {
- random.nextInt();
- } else if (n2 == 2) {
- random.nextLong();
- }
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: variable-gap terms index (divisor=" + state.termsIndexDivisor + ")");
+
+ if (doFixedGap) {
+ // if termsIndexDivisor is set to -1, we should not touch it. It means a
+ // test explicitly instructed not to load the terms index.
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: fixed-gap terms index (divisor=" + state.termsIndexDivisor + ")");
+ }
+ indexReader = new FixedGapTermsIndexReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ state.termsIndexDivisor,
+ BytesRef.getUTF8SortedAsUnicodeComparator(),
+ state.codecId, state.context);
+ } else {
+ final int n2 = random.nextInt(3);
+ if (n2 == 1) {
+ random.nextInt();
+ } else if (n2 == 2) {
+ random.nextLong();
+ }
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: variable-gap terms index (divisor=" + state.termsIndexDivisor + ")");
+ }
+ indexReader = new VariableGapTermsIndexReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ state.termsIndexDivisor,
+ state.codecId, state.context);
+
}
- if (state.termsIndexDivisor != -1) {
- state.termsIndexDivisor = _TestUtil.nextInt(random, 1, 10);
+
+ success = true;
+ } finally {
+ if (!success) {
+ postingsReader.close();
}
- indexReader = new VariableGapTermsIndexReader(state.dir,
- state.fieldInfos,
- state.segmentInfo.name,
- state.termsIndexDivisor,
- state.codecId, state.context);
}
- success = true;
- } finally {
- if (!success) {
- postingsReader.close();
- }
- }
- final int termsCacheSize = _TestUtil.nextInt(random, 1, 1024);
+ final int termsCacheSize = _TestUtil.nextInt(random, 1, 1024);
- success = false;
- try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.dir,
- state.fieldInfos,
- state.segmentInfo.name,
- postingsReader,
- state.context,
- termsCacheSize,
- state.codecId);
- success = true;
- return ret;
- } finally {
- if (!success) {
- try {
- postingsReader.close();
- } finally {
- indexReader.close();
+ success = false;
+ try {
+ fields = new BlockTermsReader(indexReader,
+ state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ postingsReader,
+ state.context,
+ termsCacheSize,
+ state.codecId);
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ postingsReader.close();
+ } finally {
+ indexReader.close();
+ }
}
}
}
+
+ return fields;
}
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SEED_EXT);
files.add(seedFileName);
- SepPostingsReaderImpl.files(segmentInfo, codecId, files);
+ SepPostingsReader.files(segmentInfo, codecId, files);
StandardPostingsReader.files(dir, segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
+ BlockTreeTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
@@ -369,8 +437,9 @@ public class MockRandomCodec extends Cod
@Override
public void getExtensions(Set<String> extensions) {
- SepPostingsWriterImpl.getExtensions(extensions);
+ SepPostingsWriter.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
+ BlockTreeTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
VariableGapTermsIndexReader.getIndexExtensions(extensions);
DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java Tue Aug 23 14:06:58 2011
@@ -40,8 +40,8 @@ import org.apache.lucene.index.codecs.Bl
import org.apache.lucene.index.codecs.TermsIndexReaderBase;
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
import org.apache.lucene.index.codecs.standard.StandardCodec;
-import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
-import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
+import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
+import org.apache.lucene.index.codecs.sep.SepPostingsReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@@ -60,7 +60,7 @@ public class MockSepCodec extends Codec
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new SepPostingsWriterImpl(state, new MockSingleIntFactory());
+ PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockSingleIntFactory());
boolean success = false;
TermsIndexWriterBase indexWriter;
@@ -92,7 +92,7 @@ public class MockSepCodec extends Codec
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
+ PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.segmentInfo,
state.context, new MockSingleIntFactory(), state.codecId);
TermsIndexReaderBase indexReader;
@@ -136,7 +136,7 @@ public class MockSepCodec extends Codec
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
- SepPostingsReaderImpl.files(segmentInfo, codecId, files);
+ SepPostingsReader.files(segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
@@ -149,7 +149,7 @@ public class MockSepCodec extends Codec
}
public static void getSepExtensions(Set<String> extensions) {
- SepPostingsWriterImpl.getExtensions(extensions);
+ SepPostingsWriter.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java Tue Aug 23 14:06:58 2011
@@ -66,6 +66,11 @@ public class MockSingleIntIndexOutput ex
out.close();
}
+ @Override
+ public String toString() {
+ return "MockSingleIntIndexOutput fp=" + out.getFilePointer();
+ }
+
private class Index extends IntIndexOutput.Index {
long fp;
long lastFP;
@@ -74,8 +79,11 @@ public class MockSingleIntIndexOutput ex
fp = out.getFilePointer();
}
@Override
- public void set(IntIndexOutput.Index other) {
- lastFP = fp = ((Index) other).fp;
+ public void copyFrom(IntIndexOutput.Index other, boolean copyLast) {
+ fp = ((Index) other).fp;
+ if (copyLast) {
+ lastFP = ((Index) other).fp;
+ }
}
@Override
public void write(IndexOutput indexOut, boolean absolute)
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java Tue Aug 23 14:06:58 2011
@@ -272,14 +272,18 @@ public class MockDirectoryWrapper extend
}
void maybeThrowIOException() throws IOException {
+ maybeThrowIOException(null);
+ }
+
+ void maybeThrowIOException(String message) throws IOException {
if (randomIOExceptionRate > 0.0) {
int number = Math.abs(randomState.nextInt() % 1000);
if (number < randomIOExceptionRate*1000) {
if (LuceneTestCase.VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception");
+ System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")"));
new Throwable().printStackTrace(System.out);
}
- throw new IOException("a random IOException");
+ throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")"));
}
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java Tue Aug 23 14:06:58 2011
@@ -147,4 +147,10 @@ public class MockIndexInputWrapper exten
public long readVLong() throws IOException {
return delegate.readVLong();
}
+
+ @Override
+ public String toString() {
+ return "MockIndexInputWrapper(" + delegate + ")";
+ }
}
+
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java Tue Aug 23 14:06:58 2011
@@ -126,7 +126,7 @@ public class MockIndexOutputWrapper exte
// Maybe throw random exception; only do this on first
// write to a new file:
first = false;
- dir.maybeThrowIOException();
+ dir.maybeThrowIOException(name);
}
}
@@ -156,4 +156,9 @@ public class MockIndexOutputWrapper exte
// TODO: we may need to check disk full here as well
dir.maybeThrowDeterministicException();
}
+
+ @Override
+ public String toString() {
+ return "MockIndexOutputWrapper(" + delegate + ")";
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Tue Aug 23 14:06:58 2011
@@ -280,7 +280,11 @@ public abstract class LuceneTestCase ext
}
swapCodec(new MockSepCodec(), cp);
- swapCodec(new PulsingCodec(codecHasParam && "Pulsing".equals(codec) ? codecParam : 1 + random.nextInt(20)), cp);
+ // TODO: make it possible to specify min/max iterms per
+ // block via CL:
+ int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100);
+ int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100);
+ swapCodec(new PulsingCodec(codecHasParam && "Pulsing".equals(codec) ? codecParam : 1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), cp);
swapCodec(new MockFixedIntBlockCodec(codecHasParam && "MockFixedIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 2000)), cp);
// baseBlockSize cannot be over 127:
swapCodec(new MockVariableIntBlockCodec(codecHasParam && "MockVariableIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 127)), cp);
@@ -307,7 +311,7 @@ public abstract class LuceneTestCase ext
cp.unregister(cp.lookup("MockFixedIntBlock"));
cp.unregister(cp.lookup("MockVariableIntBlock"));
cp.unregister(cp.lookup("MockRandom"));
- swapCodec(new PulsingCodec(1), cp);
+ swapCodec(new PulsingCodec(), cp);
cp.setDefaultFieldCodec(savedDefaultCodec);
}
@@ -322,24 +326,24 @@ public abstract class LuceneTestCase ext
}
}
- private static class TwoLongs {
- public final long l1, l2;
+ private static class ThreeLongs {
+ public final long l1, l2, l3;
- public TwoLongs(long l1, long l2) {
+ public ThreeLongs(long l1, long l2, long l3) {
this.l1 = l1;
this.l2 = l2;
+ this.l3 = l3;
}
@Override
public String toString() {
- return l1 + ":" + l2;
+ return Long.toString(l1, 16) + ":" + Long.toString(l2, 16) + ":" + Long.toString(l3, 16);
}
- public static TwoLongs fromString(String s) {
- final int i = s.indexOf(':');
- assert i != -1;
- return new TwoLongs(Long.parseLong(s.substring(0, i)),
- Long.parseLong(s.substring(1+i)));
+ public static ThreeLongs fromString(String s) {
+ String parts[] = s.split(":");
+ assert parts.length == 3;
+ return new ThreeLongs(Long.parseLong(parts[0], 16), Long.parseLong(parts[1], 16), Long.parseLong(parts[2], 16));
}
}
@@ -347,12 +351,20 @@ public abstract class LuceneTestCase ext
@Deprecated
private static List<String> testClassesRun = new ArrayList<String>();
+ private static void initRandom() {
+ assert !random.initialized;
+ staticSeed = "random".equals(TEST_SEED) ? seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l1;
+ random.setSeed(staticSeed);
+ random.initialized = true;
+ }
+
+ @Deprecated
+ private static boolean icuTested = false;
+
@BeforeClass
public static void beforeClassLuceneTestCaseJ4() {
+ initRandom();
state = State.INITIAL;
- staticSeed = "random".equals(TEST_SEED) ? seedRand.nextLong() : TwoLongs.fromString(TEST_SEED).l1;
- random.setSeed(staticSeed);
- random.initialized = true;
tempDirs.clear();
stores = Collections.synchronizedMap(new IdentityHashMap<MockDirectoryWrapper,StackTraceElement[]>());
@@ -397,7 +409,23 @@ public abstract class LuceneTestCase ext
throw new RuntimeException(e);
}
}
+
savedLocale = Locale.getDefault();
+
+ // START hack to init ICU safely before we randomize locales.
+ // ICU fails during classloading when a special Java7-only locale is the default
+ // see: http://bugs.icu-project.org/trac/ticket/8734
+ if (!icuTested) {
+ icuTested = true;
+ try {
+ Locale.setDefault(Locale.US);
+ Class.forName("com.ibm.icu.util.ULocale");
+ } catch (ClassNotFoundException cnfe) {
+ // ignore if no ICU is in classpath
+ }
+ }
+ // END hack
+
locale = TEST_LOCALE.equals("random") ? randomLocale(random) : localeForName(TEST_LOCALE);
Locale.setDefault(locale);
savedTimeZone = TimeZone.getDefault();
@@ -461,7 +489,7 @@ public abstract class LuceneTestCase ext
System.err.println("NOTE: test params are: codec=" + codecDescription +
", locale=" + locale +
", timezone=" + (timeZone == null ? "(null)" : timeZone.getID()));
- if (testsFailed) {
+ if (VERBOSE || testsFailed) {
System.err.println("NOTE: all tests run in this JVM:");
System.err.println(Arrays.toString(testClassesRun.toArray()));
System.err.println("NOTE: " + System.getProperty("os.name") + " "
@@ -543,7 +571,7 @@ public abstract class LuceneTestCase ext
@Before
public void setUp() throws Exception {
- seed = "random".equals(TEST_SEED) ? seedRand.nextLong() : TwoLongs.fromString(TEST_SEED).l2;
+ seed = "random".equals(TEST_SEED) ? seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l2;
random.setSeed(seed);
if (!testsFailed) {
assertTrue("ensure your tearDown() calls super.tearDown()!!!", (state == State.INITIAL || state == State.TEARDOWN));
@@ -754,7 +782,7 @@ public abstract class LuceneTestCase ext
* is active and {@link #RANDOM_MULTIPLIER}, but also with some random fudge.
*/
public static int atLeast(Random random, int i) {
- int min = (TEST_NIGHTLY ? 5*i : i) * RANDOM_MULTIPLIER;
+ int min = (TEST_NIGHTLY ? 3*i : i) * RANDOM_MULTIPLIER;
int max = min+(min/2);
return _TestUtil.nextInt(random, min, max);
}
@@ -770,9 +798,9 @@ public abstract class LuceneTestCase ext
* is active and {@link #RANDOM_MULTIPLIER}.
*/
public static boolean rarely(Random random) {
- int p = TEST_NIGHTLY ? 25 : 5;
+ int p = TEST_NIGHTLY ? 10 : 5;
p += (p * Math.log(RANDOM_MULTIPLIER));
- int min = 100 - Math.min(p, 90); // never more than 90
+ int min = 100 - Math.min(p, 50); // never more than 50
return random.nextInt(100) >= min;
}
@@ -1321,7 +1349,7 @@ public abstract class LuceneTestCase ext
// We get here from InterceptTestCaseEvents on the 'failed' event....
public void reportAdditionalFailureInfo() {
System.err.println("NOTE: reproduce with: ant test -Dtestcase=" + getClass().getSimpleName()
- + " -Dtestmethod=" + getName() + " -Dtests.seed=" + new TwoLongs(staticSeed, seed)
+ + " -Dtestmethod=" + getName() + " -Dtests.seed=" + new ThreeLongs(staticSeed, seed, LuceneTestCaseRunner.runnerSeed)
+ reproduceWithExtraParams());
}
@@ -1403,11 +1431,18 @@ public abstract class LuceneTestCase ext
/** optionally filters the tests to be run by TEST_METHOD */
public static class LuceneTestCaseRunner extends BlockJUnit4ClassRunner {
private List<FrameworkMethod> testMethods;
+ private static final long runnerSeed;
+ static {
+ runnerSeed = "random".equals(TEST_SEED) ? seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l3;
+ }
@Override
protected List<FrameworkMethod> computeTestMethods() {
if (testMethods != null)
return testMethods;
+
+ Random r = new Random(runnerSeed);
+
testClassesRun.add(getTestClass().getJavaClass().getSimpleName());
testMethods = new ArrayList<FrameworkMethod>();
for (Method m : getTestClass().getJavaClass().getMethods()) {
@@ -1457,6 +1492,15 @@ public abstract class LuceneTestCase ext
} catch (Exception e) { throw new RuntimeException(e); }
}
}
+ // sort the test methods first before shuffling them, so that the shuffle is consistent
+ // across different implementations that might order the methods different originally.
+ Collections.sort(testMethods, new Comparator<FrameworkMethod>() {
+ @Override
+ public int compare(FrameworkMethod f1, FrameworkMethod f2) {
+ return f1.getName().compareTo(f2.getName());
+ }
+ });
+ Collections.shuffle(testMethods, r);
return testMethods;
}
@@ -1494,6 +1538,7 @@ public abstract class LuceneTestCase ext
public LuceneTestCaseRunner(Class<?> clazz) throws InitializationError {
super(clazz);
+ // evil we cannot init our random here, because super() calls computeTestMethods!!!!;
Filter f = new Filter() {
@Override
@@ -1520,9 +1565,17 @@ public abstract class LuceneTestCase ext
RandomCodecProvider(Random random) {
this.perFieldSeed = random.nextInt();
- register(randomizCodec(random, new StandardCodec()));
+ // TODO: make it possible to specify min/max iterms per
+ // block via CL:
+ int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100);
+ int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100);
+ register(randomizCodec(random, new StandardCodec(minItemsPerBlock, maxItemsPerBlock)));
register(randomizCodec(random, new PreFlexCodec()));
- register(randomizCodec(random, new PulsingCodec( 1 + random.nextInt(20))));
+ // TODO: make it possible to specify min/max iterms per
+ // block via CL:
+ minItemsPerBlock = _TestUtil.nextInt(random, 2, 100);
+ maxItemsPerBlock = 2*(Math.max(1, minItemsPerBlock-1)) + random.nextInt(100);
+ register(randomizCodec(random, new PulsingCodec( 1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock)));
register(randomizCodec(random, new SimpleTextCodec()));
register(randomizCodec(random, new MemoryCodec()));
Collections.shuffle(knownCodecs, random);
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Tue Aug 23 14:06:58 2011
@@ -154,7 +154,7 @@ public class _TestUtil {
public static CheckIndex.Status checkIndex(Directory dir, CodecProvider codecs) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
- checker.setInfoStream(new PrintStream(bos));
+ checker.setInfoStream(new PrintStream(bos), false);
CheckIndex.Status indexStatus = checker.checkIndex(null, codecs);
if (indexStatus == null || indexStatus.clean == false) {
System.out.println("CheckIndex failed");
@@ -220,7 +220,7 @@ public class _TestUtil {
} else if (t <= 1) {
chars[i++] = (char) random.nextInt(0x80);
} else if (2 == t) {
- chars[i++] = (char) nextInt(random, 0x80, 0x800);
+ chars[i++] = (char) nextInt(random, 0x80, 0x7ff);
} else if (3 == t) {
chars[i++] = (char) nextInt(random, 0x800, 0xd7ff);
} else if (4 == t) {
@@ -281,14 +281,19 @@ public class _TestUtil {
0x2A6DF, 0x2B73F, 0x2FA1F, 0xE007F, 0xE01EF, 0xFFFFF, 0x10FFFF
};
- /** Returns random string, all codepoints within the same unicode block. */
+ /** Returns random string of length between 0-20 codepoints, all codepoints within the same unicode block. */
public static String randomRealisticUnicodeString(Random r) {
return randomRealisticUnicodeString(r, 20);
}
- /** Returns random string, all codepoints within the same unicode block. */
+ /** Returns random string of length up to maxLength codepoints , all codepoints within the same unicode block. */
public static String randomRealisticUnicodeString(Random r, int maxLength) {
- final int end = r.nextInt(maxLength);
+ return randomRealisticUnicodeString(r, 0, 20);
+ }
+
+ /** Returns random string of length between min and max codepoints, all codepoints within the same unicode block. */
+ public static String randomRealisticUnicodeString(Random r, int minLength, int maxLength) {
+ final int end = minLength + r.nextInt(maxLength);
final int block = r.nextInt(blockStarts.length);
StringBuilder sb = new StringBuilder();
for (int i = 0; i < end; i++)
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Tue Aug 23 14:06:58 2011
@@ -92,10 +92,6 @@ public class TestExternalCodecs extends
@Override
public void close() {
}
-
- @Override
- public void loadTermsIndex(int indexDivisor) {
- }
}
static class RAMField extends Terms {
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java Tue Aug 23 14:06:58 2011
@@ -86,7 +86,7 @@ public class TestSearchForDuplicates ext
}
IndexWriter writer = new IndexWriter(directory, conf);
if (VERBOSE) {
- System.out.println("TEST: now build index");
+ System.out.println("TEST: now build index MAX_DOCS=" + MAX_DOCS);
writer.setInfoStream(System.out);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/Test2BPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/Test2BPostings.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/Test2BPostings.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/Test2BPostings.java Tue Aug 23 14:06:58 2011
@@ -70,14 +70,16 @@ public class Test2BPostings extends Luce
final int numDocs = (Integer.MAX_VALUE / 26) + 1;
for (int i = 0; i < numDocs; i++) {
w.addDocument(doc);
- if (i % 100000 == 0) {
+ if (VERBOSE && i % 100000 == 0) {
System.out.println(i + " of " + numDocs + "...");
}
}
w.optimize();
w.close();
CheckIndex ci = new CheckIndex(dir);
- ci.setInfoStream(System.out);
+ if (VERBOSE) {
+ ci.setInfoStream(System.out);
+ }
ci.checkIndex();
dir.close();
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/Test2BTerms.java Tue Aug 23 14:06:58 2011
@@ -155,9 +155,9 @@ public class Test2BTerms extends LuceneT
List<BytesRef> savedTerms = null;
MockDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BTerms"));
+ //MockDirectoryWrapper dir = newFSDirectory(new File("/p/lucene/indices/2bindex"));
dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);
dir.setCheckIndexOnClose(false); // don't double-checkindex
- //Directory dir = newFSDirectory(new File("/p/lucene/indices/2bindex"));
if (true) {
@@ -169,6 +169,7 @@ public class Test2BTerms extends LuceneT
.setMergePolicy(newLogMergePolicy(false, 10))
.setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+ w.setInfoStream(VERBOSE ? System.out : null);
MergePolicy mp = w.getConfig().getMergePolicy();
if (mp instanceof LogByteSizeMergePolicy) {
// 1 petabyte:
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestCodecs.java Tue Aug 23 14:06:58 2011
@@ -40,8 +40,6 @@ import org.apache.lucene.search.PhraseQu
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IOContext.Context;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
@@ -504,6 +502,9 @@ public class TestCodecs extends LuceneTe
}
// Test seek to non-existent terms:
+ if (VERBOSE) {
+ System.out.println("TEST: seek non-exist terms");
+ }
for(int i=0;i<100;i++) {
final String text2 = _TestUtil.randomUnicodeString(random) + ".";
status = termsEnum.seekCeil(new BytesRef(text2));
@@ -512,6 +513,9 @@ public class TestCodecs extends LuceneTe
}
// Seek to each term, backwards:
+ if (VERBOSE) {
+ System.out.println("TEST: seek terms backwards");
+ }
for(int i=field.terms.length-1;i>=0;i--) {
assertEquals(Thread.currentThread().getName() + ": field=" + field.fieldInfo.name + " term=" + field.terms[i].text2, TermsEnum.SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef(field.terms[i].text2)));
assertEquals(field.terms[i].docs.length, termsEnum.docFreq());
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java Tue Aug 23 14:06:58 2011
@@ -462,7 +462,7 @@ public class TestDocTermOrds extends Luc
*/
if (VERBOSE) {
- System.out.println("TEST: verify prefix=" + prefixRef.utf8ToString());
+ System.out.println("TEST: verify prefix=" + (prefixRef==null ? "null" : prefixRef.utf8ToString()));
System.out.println("TEST: all TERMS:");
TermsEnum allTE = MultiFields.getTerms(r, "field").iterator();
int ord = 0;
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java Tue Aug 23 14:06:58 2011
@@ -194,7 +194,8 @@ public class TestDocsAndPositions extend
public void testRandomDocs() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
+ writer.w.setInfoStream(VERBOSE ? System.out : null);
int numDocs = atLeast(49);
int max = 15678;
int term = random.nextInt(max);
@@ -290,7 +291,7 @@ public class TestDocsAndPositions extend
writer.addDocument(doc);
}
- // now do seaches
+ // now do searches
IndexReader reader = writer.getReader();
writer.close();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java Tue Aug 23 14:06:58 2011
@@ -16,7 +16,6 @@ package org.apache.lucene.index;
* License for the specific language governing permissions and limitations under
* the License.
*/
-import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@@ -24,6 +23,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
@@ -41,8 +41,8 @@ public class TestDocumentsWriterDeleteQu
}
DeleteSlice slice1 = queue.newSlice();
DeleteSlice slice2 = queue.newSlice();
- BufferedDeletes bd1 = new BufferedDeletes(false);
- BufferedDeletes bd2 = new BufferedDeletes(false);
+ BufferedDeletes bd1 = new BufferedDeletes();
+ BufferedDeletes bd2 = new BufferedDeletes();
int last1 = 0;
int last2 = 0;
Set<Term> uniqueValues = new HashSet<Term>();
@@ -70,8 +70,13 @@ public class TestDocumentsWriterDeleteQu
}
assertEquals(uniqueValues, bd1.terms.keySet());
assertEquals(uniqueValues, bd2.terms.keySet());
- assertEquals(uniqueValues, new HashSet<Term>(Arrays.asList(queue
- .freezeGlobalBuffer(null).terms)));
+ HashSet<Term> frozenSet = new HashSet<Term>();
+ for (Term t : queue.freezeGlobalBuffer(null).termsIterable()) {
+ BytesRef bytesRef = new BytesRef();
+ bytesRef.copy(t.bytes);
+ frozenSet.add(new Term(t.field, bytesRef));
+ }
+ assertEquals(uniqueValues, frozenSet);
assertEquals("num deletes must be 0 after freeze", 0, queue
.numGlobalTermDeletes());
}
@@ -129,7 +134,7 @@ public class TestDocumentsWriterDeleteQu
if (random.nextInt(5) == 0) {
FrozenBufferedDeletes freezeGlobalBuffer = queue
.freezeGlobalBuffer(null);
- assertEquals(termsSinceFreeze, freezeGlobalBuffer.terms.length);
+ assertEquals(termsSinceFreeze, freezeGlobalBuffer.termCount);
assertEquals(queriesSinceFreeze, freezeGlobalBuffer.queries.length);
queriesSinceFreeze = 0;
termsSinceFreeze = 0;
@@ -168,10 +173,17 @@ public class TestDocumentsWriterDeleteQu
assertEquals(uniqueValues, deletes.terms.keySet());
}
queue.tryApplyGlobalSlice();
- assertEquals(uniqueValues, new HashSet<Term>(Arrays.asList(queue
- .freezeGlobalBuffer(null).terms)));
+ Set<Term> frozenSet = new HashSet<Term>();
+ for (Term t : queue.freezeGlobalBuffer(null).termsIterable()) {
+ BytesRef bytesRef = new BytesRef();
+ bytesRef.copy(t.bytes);
+ frozenSet.add(new Term(t.field, bytesRef));
+ }
assertEquals("num deletes must be 0 after freeze", 0, queue
.numGlobalTermDeletes());
+ assertEquals(uniqueValues.size(), frozenSet.size());
+ assertEquals(uniqueValues, frozenSet);
+
}
private static class UpdateThread extends Thread {
@@ -188,7 +200,7 @@ public class TestDocumentsWriterDeleteQu
this.index = index;
this.ids = ids;
this.slice = queue.newSlice();
- deletes = new BufferedDeletes(false);
+ deletes = new BufferedDeletes();
this.latch = latch;
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Tue Aug 23 14:06:58 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.search.similarities.DefaultSimilarity;
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -169,7 +168,7 @@ public class TestIndexFileDeleter extend
Set<String> dif = difFiles(files, files2);
if (!Arrays.equals(files, files2)) {
- fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.length-files.length) + " files but only deleted " + (filesPre.length - files2.length) + "; expected files:\n " + asString(files) + "\n actual files:\n " + asString(files2)+"\ndif: "+dif);
+ fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.length-files.length) + " files but only deleted " + (filesPre.length - files2.length) + "; expected files:\n " + asString(files) + "\n actual files:\n " + asString(files2)+"\ndiff: "+dif);
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Tue Aug 23 14:06:58 2011
@@ -69,7 +69,7 @@ public class TestIndexWriterConfig exten
assertEquals(IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR, conf.getReaderTermsIndexDivisor());
assertEquals(TieredMergePolicy.class, conf.getMergePolicy().getClass());
assertEquals(ThreadAffinityDocumentsWriterThreadPool.class, conf.getIndexerThreadPool().getClass());
- assertNull(conf.getFlushPolicy());
+ assertEquals(FlushByRamOrCountsPolicy.class, conf.getFlushPolicy().getClass());
assertEquals(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.getRAMPerThreadHardLimitMB());
// Sanity check - validate that all getters are covered.
Set<String> getters = new HashSet<String>();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue Aug 23 14:06:58 2011
@@ -23,6 +23,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -72,8 +74,15 @@ public class TestIndexWriterDelete exten
Term term = new Term("city", "Amsterdam");
int hitCount = getHitCount(dir, term);
assertEquals(1, hitCount);
+ if (VERBOSE) {
+ System.out.println("\nTEST: now delete by term=" + term);
+ }
modifier.deleteDocuments(term);
modifier.commit();
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: now getHitCount");
+ }
hitCount = getHitCount(dir, term);
assertEquals(0, hitCount);
@@ -551,7 +560,9 @@ public class TestIndexWriterDelete exten
if (!success) {
// Must force the close else the writer can have
// open files which cause exc in MockRAMDir.close
-
+ if (VERBOSE) {
+ System.out.println("TEST: now rollback");
+ }
modifier.rollback();
}
@@ -942,4 +953,137 @@ public class TestIndexWriterDelete exten
w.close();
dir.close();
}
+
+ // LUCENE-3340: make sure deletes that we don't apply
+ // during flush (ie are just pushed into the stream) are
+ // in fact later flushed due to their RAM usage:
+ public void testFlushPushedDeletesByRAM() throws Exception {
+ Directory dir = newDirectory();
+ // Cannot use RandomIndexWriter because we don't want to
+ // ever call commit() for this test:
+ // note: tiny rambuffer used, as with a 1MB buffer the test is too slow (flush @ 128,999)
+ IndexWriter w = new IndexWriter(dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setRAMBufferSizeMB(0.2f).setMaxBufferedDocs(1000).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false));
+ w.setInfoStream(VERBOSE ? System.out : null);
+ int count = 0;
+ while(true) {
+ Document doc = new Document();
+ doc.add(new Field("id", count+"", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ final Term delTerm;
+ if (count == 1010) {
+ // This is the only delete that applies
+ delTerm = new Term("id", ""+0);
+ } else {
+ // These get buffered, taking up RAM, but delete
+ // nothing when applied:
+ delTerm = new Term("id", "x" + count);
+ }
+ w.updateDocument(delTerm, doc);
+ // Eventually segment 0 should get a del docs:
+ if (dir.fileExists("_0_1.del")) {
+ if (VERBOSE) {
+ System.out.println("TEST: deletes created @ count=" + count);
+ }
+ break;
+ }
+ count++;
+
+ // Today we applyDeletes @ count=21553; even if we make
+ // sizable improvements to RAM efficiency of buffered
+ // del term we're unlikely to go over 100K:
+ if (count > 100000) {
+ fail("delete's were not applied");
+ }
+ }
+ w.close();
+ dir.close();
+ }
+
+ // LUCENE-3340: make sure deletes that we don't apply
+ // during flush (ie are just pushed into the stream) are
+ // in fact later flushed due to their RAM usage:
+ public void testFlushPushedDeletesByCount() throws Exception {
+ Directory dir = newDirectory();
+ // Cannot use RandomIndexWriter because we don't want to
+ // ever call commit() for this test:
+ final int flushAtDelCount = atLeast(1020);
+ IndexWriter w = new IndexWriter(dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMaxBufferedDeleteTerms(flushAtDelCount).setMaxBufferedDocs(1000).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false));
+ w.setInfoStream(VERBOSE ? System.out : null);
+ int count = 0;
+ while(true) {
+ Document doc = new Document();
+ doc.add(new Field("id", count+"", Field.Store.NO, Field.Index.NOT_ANALYZED));
+ final Term delTerm;
+ if (count == 1010) {
+ // This is the only delete that applies
+ delTerm = new Term("id", ""+0);
+ } else {
+ // These get buffered, taking up RAM, but delete
+ // nothing when applied:
+ delTerm = new Term("id", "x" + count);
+ }
+ w.updateDocument(delTerm, doc);
+ // Eventually segment 0 should get a del docs:
+ if (dir.fileExists("_0_1.del")) {
+ break;
+ }
+ count++;
+ if (count > flushAtDelCount) {
+ fail("delete's were not applied at count=" + flushAtDelCount);
+ }
+ }
+ w.close();
+ dir.close();
+ }
+
+ // Make sure buffered (pushed) deletes don't use up so
+ // much RAM that it forces long tail of tiny segments:
+ public void testApplyDeletesOnFlush() throws Exception {
+ Directory dir = newDirectory();
+ // Cannot use RandomIndexWriter because we don't want to
+ // ever call commit() for this test:
+ final AtomicInteger docsInSegment = new AtomicInteger();
+ final AtomicBoolean closing = new AtomicBoolean();
+ final AtomicBoolean sawAfterFlush = new AtomicBoolean();
+ IndexWriter w = new IndexWriter(dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setRAMBufferSizeMB(0.5).setMaxBufferedDocs(-1).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false)) {
+ @Override
+ public void doAfterFlush() {
+ assertTrue("only " + docsInSegment.get() + " in segment", closing.get() || docsInSegment.get() >= 7);
+ docsInSegment.set(0);
+ sawAfterFlush.set(true);
+ }
+ };
+ w.setInfoStream(VERBOSE ? System.out : null);
+ int id = 0;
+ while(true) {
+ StringBuilder sb = new StringBuilder();
+ for(int termIDX=0;termIDX<100;termIDX++) {
+ sb.append(' ').append(_TestUtil.randomRealisticUnicodeString(random));
+ }
+ if (id == 500) {
+ w.deleteDocuments(new Term("id", "0"));
+ }
+ Document doc = new Document();
+ doc.add(newField("id", ""+id, Field.Index.NOT_ANALYZED));
+ doc.add(newField("body", sb.toString(), Field.Index.ANALYZED));
+ w.updateDocument(new Term("id", ""+id), doc);
+ docsInSegment.incrementAndGet();
+ if (dir.fileExists("_0_1.del")) {
+ if (VERBOSE) {
+ System.out.println("TEST: deletes created @ id=" + id);
+ }
+ break;
+ }
+ id++;
+ }
+ closing.set(true);
+ assertTrue(sawAfterFlush.get());
+ w.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java Tue Aug 23 14:06:58 2011
@@ -221,6 +221,8 @@ public class TestIndexWriterMerging exte
setMergePolicy(newLogMergePolicy(50))
);
+ writer.setInfoStream(VERBOSE ? System.out : null);
+
Document document = new Document();
document = new Document();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Tue Aug 23 14:06:58 2011
@@ -28,7 +28,6 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
@@ -493,7 +492,7 @@ public class TestIndexWriterOnDiskFull e
fail("fake disk full IOExceptions not hit");
} catch (IOException ioe) {
// expected
- assertTrue(ftdm.didFail1);
+ assertTrue(ftdm.didFail1 || ftdm.didFail2);
}
_TestUtil.checkIndex(dir);
ftdm.clearDoFail();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Tue Aug 23 14:06:58 2011
@@ -51,7 +51,7 @@ public class TestIndexWriterOnJRECrash e
public void testNRTThreads() throws Exception {
String vendor = Constants.JAVA_VENDOR;
assumeTrue(vendor + " JRE not supported.",
- vendor.startsWith("Sun") || vendor.startsWith("Apple"));
+ vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple"));
// if we are not the fork
if (System.getProperty("tests.crashmode") == null) {
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Tue Aug 23 14:06:58 2011
@@ -74,7 +74,7 @@ public class TestIndexWriterReader exten
for (int i = 0; i < 97 ; i++) {
IndexReader reader = writer.getReader();
if (i == 0) {
- writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
+ writer.addDocument(DocHelper.createDocument(i, "x", 1 + random.nextInt(5)));
} else {
int previous = random.nextInt(i);
// a check if the reader is current here could fail since there might be
@@ -83,10 +83,10 @@ public class TestIndexWriterReader exten
case 0:
case 1:
case 2:
- writer.addDocument(createDocument(i, "x", 1 + random.nextInt(5)));
+ writer.addDocument(DocHelper.createDocument(i, "x", 1 + random.nextInt(5)));
break;
case 3:
- writer.updateDocument(new Term("id", "" + previous), createDocument(
+ writer.updateDocument(new Term("id", "" + previous), DocHelper.createDocument(
previous, "x", 1 + random.nextInt(5)));
break;
case 4:
@@ -105,7 +105,7 @@ public class TestIndexWriterReader exten
iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
writer = new IndexWriter(dir1, iwc);
assertTrue(reader.isCurrent());
- writer.addDocument(createDocument(1, "x", 1+random.nextInt(5)));
+ writer.addDocument(DocHelper.createDocument(1, "x", 1+random.nextInt(5)));
assertTrue(reader.isCurrent()); // segments in ram but IW is different to the readers one
writer.close();
assertFalse(reader.isCurrent()); // segments written
@@ -422,7 +422,7 @@ public class TestIndexWriterReader exten
addDir = newDirectory();
IndexWriter writer = new IndexWriter(addDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
for (int i = 0; i < NUM_INIT_DOCS; i++) {
- Document doc = createDocument(i, "addindex", 4);
+ Document doc = DocHelper.createDocument(i, "addindex", 4);
writer.addDocument(doc);
}
@@ -543,7 +543,7 @@ public class TestIndexWriterReader exten
assertEquals(r2.maxDoc(), 100);
// add 100 documents
for (int x = 10000; x < 10000 + 100; x++) {
- Document d = createDocument(x, "index1", 5);
+ Document d = DocHelper.createDocument(x, "index1", 5);
writer.addDocument(d);
}
writer.flush(false, true);
@@ -575,25 +575,7 @@ public class TestIndexWriterReader exten
dir1.close();
}
-
-
- public static Document createDocument(int n, String indexName, int numFields) {
- StringBuilder sb = new StringBuilder();
- Document doc = new Document();
- doc.add(new Field("id", Integer.toString(n), Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- doc.add(new Field("indexname", indexName, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- sb.append("a");
- sb.append(n);
- doc.add(new Field("field1", sb.toString(), Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- sb.append(" b");
- sb.append(n);
- for (int i = 1; i < numFields; i++) {
- doc.add(new Field("field" + (i + 1), sb.toString(), Store.YES,
- Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- }
- return doc;
- }
-
+
/*
* Delete a document by term and return the doc id
*
@@ -609,7 +591,7 @@ public class TestIndexWriterReader exten
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setMergePolicy(new LogDocMergePolicy()));
for (int i = 0; i < 100; i++) {
- w.addDocument(createDocument(i, indexName, 4));
+ w.addDocument(DocHelper.createDocument(i, indexName, 4));
if (multiSegment && (i % 10) == 0) {
}
}
@@ -622,7 +604,7 @@ public class TestIndexWriterReader exten
public static void createIndexNoClose(boolean multiSegment, String indexName,
IndexWriter w) throws IOException {
for (int i = 0; i < 100; i++) {
- w.addDocument(createDocument(i, indexName, 4));
+ w.addDocument(DocHelper.createDocument(i, indexName, 4));
}
if (!multiSegment) {
w.optimize();
@@ -662,14 +644,14 @@ public class TestIndexWriterReader exten
int num = atLeast(100);
for (int i = 0; i < num; i++) {
- writer.addDocument(createDocument(i, "test", 4));
+ writer.addDocument(DocHelper.createDocument(i, "test", 4));
}
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
assertTrue(warmer.warmCount > 0);
final int count = warmer.warmCount;
- writer.addDocument(createDocument(17, "test", 4));
+ writer.addDocument(DocHelper.createDocument(17, "test", 4));
writer.optimize();
assertTrue(warmer.warmCount > count);
@@ -695,7 +677,7 @@ public class TestIndexWriterReader exten
assertEquals(100, r1.numDocs());
for (int i = 0; i < 10; i++) {
- writer.addDocument(createDocument(i, "test", 4));
+ writer.addDocument(DocHelper.createDocument(i, "test", 4));
}
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync();
@@ -857,7 +839,7 @@ public class TestIndexWriterReader exten
do {
try {
for(int docUpto=0;docUpto<10;docUpto++) {
- writer.addDocument(createDocument(10*count+docUpto, "test", 4));
+ writer.addDocument(DocHelper.createDocument(10*count+docUpto, "test", 4));
}
count++;
final int limit = count*10;
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Tue Aug 23 14:06:58 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -408,4 +409,76 @@ public class TestIndexWriterWithThreads
public void testIOExceptionDuringWriteSegmentWithThreadsOnlyOnce() throws Exception {
_testMultipleThreadsFailure(new FailOnlyInWriteSegment(true));
}
+
+ // LUCENE-3365: Test adding two documents with the same field from two different IndexWriters
+ // that we attempt to open at the same time. As long as the first IndexWriter completes
+ // and closes before the second IndexWriter time's out trying to get the Lock,
+ // we should see both documents
+ public void testOpenTwoIndexWritersOnDifferentThreads() throws IOException, InterruptedException {
+ final MockDirectoryWrapper dir = newDirectory();
+ CountDownLatch oneIWConstructed = new CountDownLatch(1);
+ DelayedIndexAndCloseRunnable thread1 = new DelayedIndexAndCloseRunnable(
+ dir, oneIWConstructed);
+ DelayedIndexAndCloseRunnable thread2 = new DelayedIndexAndCloseRunnable(
+ dir, oneIWConstructed);
+
+ thread1.start();
+ thread2.start();
+ oneIWConstructed.await();
+
+ thread1.startIndexing();
+ thread2.startIndexing();
+
+ thread1.join();
+ thread2.join();
+
+ assertFalse("Failed due to: " + thread1.failure, thread1.failed);
+ assertFalse("Failed due to: " + thread2.failure, thread2.failed);
+ // now verify that we have two documents in the index
+ IndexReader reader = IndexReader.open(dir, true);
+ assertEquals("IndexReader should have one document per thread running", 2,
+ reader.numDocs());
+
+ reader.close();
+ dir.close();
+ }
+
+ static class DelayedIndexAndCloseRunnable extends Thread {
+ private final Directory dir;
+ boolean failed = false;
+ Throwable failure = null;
+ private final CountDownLatch startIndexing = new CountDownLatch(1);
+ private CountDownLatch iwConstructed;
+
+ public DelayedIndexAndCloseRunnable(Directory dir,
+ CountDownLatch iwConstructed) {
+ this.dir = dir;
+ this.iwConstructed = iwConstructed;
+ }
+
+ public void startIndexing() {
+ this.startIndexing.countDown();
+ }
+
+ @Override
+ public void run() {
+ try {
+ Document doc = new Document();
+ Field field = newField("field", "testData", Field.Store.YES,
+ Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
+ doc.add(field);
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ iwConstructed.countDown();
+ startIndexing.await();
+ writer.addDocument(doc);
+ writer.close();
+ } catch (Throwable e) {
+ failed = true;
+ failure = e;
+ failure.printStackTrace(System.out);
+ return;
+ }
+ }
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Tue Aug 23 14:06:58 2011
@@ -355,7 +355,7 @@ public class TestLongPostings extends Lu
}
if (VERBOSE) {
- System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1);
+ System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1 + " term=" + term);
}
final DocsEnum postings = MultiFields.getTermDocsEnum(r, null, "field", new BytesRef(term));
@@ -426,7 +426,7 @@ public class TestLongPostings extends Lu
if (random.nextInt(6) == 3) {
final int freq = postings.freq();
- assertTrue(freq >=1 && freq <= 4);
+ assertTrue("got invalid freq=" + freq, freq >=1 && freq <= 4);
}
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMultiFields.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMultiFields.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMultiFields.java Tue Aug 23 14:06:58 2011
@@ -29,10 +29,15 @@ public class TestMultiFields extends Luc
int num = atLeast(2);
for (int iter = 0; iter < num; iter++) {
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + iter);
+ }
+
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(NoMergePolicy.COMPOUND_FILES));
_TestUtil.keepFullyDeletedSegments(w);
+ w.setInfoStream(VERBOSE ? System.out : null);
Map<BytesRef,List<Integer>> docs = new HashMap<BytesRef,List<Integer>>();
Set<Integer> deleted = new HashSet<Integer>();
@@ -46,6 +51,9 @@ public class TestMultiFields extends Luc
doc.add(id);
boolean onlyUniqueTerms = random.nextBoolean();
+ if (VERBOSE) {
+ System.out.println("TEST: onlyUniqueTerms=" + onlyUniqueTerms + " numDocs=" + numDocs);
+ }
Set<BytesRef> uniqueTerms = new HashSet<BytesRef>();
for(int i=0;i<numDocs;i++) {
@@ -74,21 +82,33 @@ public class TestMultiFields extends Luc
int delID = random.nextInt(i);
deleted.add(delID);
w.deleteDocuments(new Term("id", ""+delID));
+ if (VERBOSE) {
+ System.out.println("TEST: delete " + delID);
+ }
}
}
if (VERBOSE) {
List<BytesRef> termsList = new ArrayList<BytesRef>(uniqueTerms);
Collections.sort(termsList, BytesRef.getUTF8SortedAsUTF16Comparator());
- System.out.println("UTF16 order:");
+ System.out.println("TEST: terms in UTF16 order:");
for(BytesRef b : termsList) {
- System.out.println(" " + UnicodeUtil.toHexString(b.utf8ToString()));
+ System.out.println(" " + UnicodeUtil.toHexString(b.utf8ToString()) + " " + b);
+ for(int docID : docs.get(b)) {
+ if (deleted.contains(docID)) {
+ System.out.println(" " + docID + " (deleted)");
+ } else {
+ System.out.println(" " + docID);
+ }
+ }
}
}
IndexReader reader = w.getReader();
w.close();
- //System.out.println("TEST reader=" + reader);
+ if (VERBOSE) {
+ System.out.println("TEST: reader=" + reader);
+ }
Bits liveDocs = MultiFields.getLiveDocs(reader);
for(int delDoc : deleted) {
@@ -99,7 +119,7 @@ public class TestMultiFields extends Luc
for(int i=0;i<100;i++) {
BytesRef term = terms.get(random.nextInt(terms.size()));
if (VERBOSE) {
- System.out.println("TEST: seek to term= "+ UnicodeUtil.toHexString(term.utf8ToString()));
+ System.out.println("TEST: seek term="+ UnicodeUtil.toHexString(term.utf8ToString()) + " " + term);
}
DocsEnum docsEnum = terms2.docs(liveDocs, term, null);
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java?rev=1160700&r1=1160699&r2=1160700&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java Tue Aug 23 14:06:58 2011
@@ -91,7 +91,7 @@ public class TestNRTReaderWithThreads ex
//int n = random.nextInt(2);
if (type == 0) {
int i = seq.addAndGet(1);
- Document doc = TestIndexWriterReader.createDocument(i, "index1", 10);
+ Document doc = DocHelper.createDocument(i, "index1", 10);
writer.addDocument(doc);
addCount++;
} else if (type == 1) {