You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/04/21 20:36:34 UTC
svn commit: r1588953 - in /lucene/dev/trunk/lucene/misc/src:
java/org/apache/lucene/index/sorter/ test/org/apache/lucene/index/sorter/
Author: rmuir
Date: Mon Apr 21 18:36:34 2014
New Revision: 1588953
URL: http://svn.apache.org/r1588953
Log:
LUCENE-5623: fix bug in earlyterminatingcollector, fix test to be reproducible and more evil
Modified:
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminatingSortingCollector.java
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminatingSortingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminatingSortingCollector.java?rev=1588953&r1=1588952&r2=1588953&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminatingSortingCollector.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminatingSortingCollector.java Mon Apr 21 18:36:34 2014
@@ -70,8 +70,6 @@ public class EarlyTerminatingSortingColl
/** Number of documents to collect in each segment */
protected final int numDocsToCollect;
- private int numCollected;
-
/**
* Create a new {@link EarlyTerminatingSortingCollector} instance.
*
@@ -98,6 +96,7 @@ public class EarlyTerminatingSortingColl
if (SortingMergePolicy.isSorted(context.reader(), sort)) {
// segment is sorted, can early-terminate
return new FilterLeafCollector(super.getLeafCollector(context)) {
+ private int numCollected;
@Override
public void collect(int doc) throws IOException {
Modified: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java?rev=1588953&r1=1588952&r2=1588953&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java (original)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java Mon Apr 21 18:36:34 2014
@@ -33,9 +33,12 @@ import org.apache.lucene.index.AtomicRea
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
@@ -70,7 +73,7 @@ public class TestEarlyTermination extend
return doc;
}
- private void createRandomIndexes(int maxSegments) throws IOException {
+ private void createRandomIndex() throws IOException {
dir = newDirectory();
numDocs = atLeast(150);
final int numTerms = TestUtil.nextInt(random(), 1, numDocs / 5);
@@ -81,8 +84,10 @@ public class TestEarlyTermination extend
terms = new ArrayList<>(randomTerms);
final long seed = random().nextLong();
final IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(new Random(seed)));
+ iwc.setMergeScheduler(new SerialMergeScheduler()); // for reproducible tests
iwc.setMergePolicy(TestSortingMergePolicy.newSortingMergePolicy(sort));
iw = new RandomIndexWriter(new Random(seed), dir, iwc);
+ iw.setDoRandomForceMerge(false); // don't do this, it may happen anyway with MockRandomMP
for (int i = 0; i < numDocs; ++i) {
final Document doc = randomDocument();
iw.addDocument(doc);
@@ -94,56 +99,70 @@ public class TestEarlyTermination extend
iw.deleteDocuments(new Term("s", term));
}
}
+ if (random().nextBoolean()) {
+ iw.forceMerge(5);
+ }
reader = iw.getReader();
}
-
- @Override
- public void tearDown() throws Exception {
+
+ private void closeIndex() throws IOException {
reader.close();
iw.shutdown();
dir.close();
- super.tearDown();
}
public void testEarlyTermination() throws IOException {
- createRandomIndexes(5);
- final int numHits = TestUtil.nextInt(random(), 1, numDocs / 10);
- final Sort sort = new Sort(new SortField("ndv1", SortField.Type.LONG, false));
- final boolean fillFields = random().nextBoolean();
- final boolean trackDocScores = random().nextBoolean();
- final boolean trackMaxScore = random().nextBoolean();
- final boolean inOrder = random().nextBoolean();
- final TopFieldCollector collector1 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder);
- final TopFieldCollector collector2 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder);
-
- final IndexSearcher searcher = newSearcher(reader);
- final int iters = atLeast(5);
+ final int iters = atLeast(8);
for (int i = 0; i < iters; ++i) {
- final TermQuery query = new TermQuery(new Term("s", RandomPicks.randomFrom(random(), terms)));
- searcher.search(query, collector1);
- searcher.search(query, new EarlyTerminatingSortingCollector(collector2, sort, numHits));
+ createRandomIndex();
+ for (int j = 0; j < iters; ++j) {
+ final IndexSearcher searcher = newSearcher(reader);
+ final int numHits = TestUtil.nextInt(random(), 1, numDocs);
+ final Sort sort = new Sort(new SortField("ndv1", SortField.Type.LONG, false));
+ final boolean fillFields = random().nextBoolean();
+ final boolean trackDocScores = random().nextBoolean();
+ final boolean trackMaxScore = random().nextBoolean();
+ final boolean inOrder = random().nextBoolean();
+ final TopFieldCollector collector1 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder);
+ final TopFieldCollector collector2 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder);
+
+ final Query query;
+ if (random().nextBoolean()) {
+ query = new TermQuery(new Term("s", RandomPicks.randomFrom(random(), terms)));
+ } else {
+ query = new MatchAllDocsQuery();
+ }
+ searcher.search(query, collector1);
+ searcher.search(query, new EarlyTerminatingSortingCollector(collector2, sort, numHits));
+ assertTrue(collector1.getTotalHits() >= collector2.getTotalHits());
+ assertTopDocsEquals(collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs);
+ }
+ closeIndex();
}
- assertTrue(collector1.getTotalHits() >= collector2.getTotalHits());
- assertTopDocsEquals(collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs);
}
public void testEarlyTerminationDifferentSorter() throws IOException {
- // test that the collector works correctly when the index was sorted by a
- // different sorter than the one specified in the ctor.
- createRandomIndexes(5);
- final int numHits = TestUtil.nextInt(random(), 1, numDocs / 10);
- final Sort sort = new Sort(new SortField("ndv2", SortField.Type.LONG, false));
- final boolean fillFields = random().nextBoolean();
- final boolean trackDocScores = random().nextBoolean();
- final boolean trackMaxScore = random().nextBoolean();
- final boolean inOrder = random().nextBoolean();
- final TopFieldCollector collector1 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder);
- final TopFieldCollector collector2 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder);
-
- final IndexSearcher searcher = newSearcher(reader);
- final int iters = atLeast(5);
+ createRandomIndex();
+ final int iters = atLeast(3);
for (int i = 0; i < iters; ++i) {
- final TermQuery query = new TermQuery(new Term("s", RandomPicks.randomFrom(random(), terms)));
+ final IndexSearcher searcher = newSearcher(reader);
+ // test that the collector works correctly when the index was sorted by a
+ // different sorter than the one specified in the ctor.
+ final int numHits = TestUtil.nextInt(random(), 1, numDocs);
+ final Sort sort = new Sort(new SortField("ndv2", SortField.Type.LONG, false));
+ final boolean fillFields = random().nextBoolean();
+ final boolean trackDocScores = random().nextBoolean();
+ final boolean trackMaxScore = random().nextBoolean();
+ final boolean inOrder = random().nextBoolean();
+ final TopFieldCollector collector1 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder);
+ final TopFieldCollector collector2 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder);
+
+ final Query query;
+ if (random().nextBoolean()) {
+ query = new TermQuery(new Term("s", RandomPicks.randomFrom(random(), terms)));
+ } else {
+ query = new MatchAllDocsQuery();
+ }
searcher.search(query, collector1);
Sort different = new Sort(new SortField("ndv2", SortField.Type.LONG));
searcher.search(query, new EarlyTerminatingSortingCollector(collector2, different, numHits) {
@@ -154,9 +173,10 @@ public class TestEarlyTermination extend
return ret;
}
});
+ assertTrue(collector1.getTotalHits() >= collector2.getTotalHits());
+ assertTopDocsEquals(collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs);
}
- assertTrue(collector1.getTotalHits() >= collector2.getTotalHits());
- assertTopDocsEquals(collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs);
+ closeIndex();
}
private static void assertTopDocsEquals(ScoreDoc[] scoreDocs1, ScoreDoc[] scoreDocs2) {
Modified: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java?rev=1588953&r1=1588952&r2=1588953&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java (original)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java Mon Apr 21 18:36:34 2014
@@ -71,18 +71,22 @@ public class TestSortingMergePolicy exte
}
static MergePolicy newSortingMergePolicy(Sort sort) {
- // create a MP with a low merge factor so that many merges happen
+ // usually create a MP with a low merge factor so that many merges happen
MergePolicy mp;
- if (random().nextBoolean()) {
+ int thingToDo = random().nextInt(3);
+ if (thingToDo == 0) {
TieredMergePolicy tmp = newTieredMergePolicy(random());
final int numSegs = TestUtil.nextInt(random(), 3, 5);
tmp.setSegmentsPerTier(numSegs);
tmp.setMaxMergeAtOnce(TestUtil.nextInt(random(), 2, numSegs));
mp = tmp;
- } else {
+ } else if (thingToDo == 1) {
LogMergePolicy lmp = newLogMergePolicy(random());
lmp.setMergeFactor(TestUtil.nextInt(random(), 3, 5));
mp = lmp;
+ } else {
+ // just a regular random one from LTC (could be alcoholic etc)
+ mp = newMergePolicy();
}
// wrap it with a sorting mp
return new SortingMergePolicy(mp, sort);