You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2016/06/16 17:48:19 UTC
[2/3] lucene-solr:branch_5_5: LUCENE-7132: BooleanQuery sometimes
assigned the wrong score when ranges of documents had only one clause
matching while other ranges had more than one clause matchng
LUCENE-7132: BooleanQuery sometimes assigned the wrong score when ranges of documents had only one clause matching while other ranges had more than one clause matchng
(Cherry-picked from commit 5dfaf0392fcd3b7e4b529dce0cd1035b766880a7)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/77844e25
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/77844e25
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/77844e25
Branch: refs/heads/branch_5_5
Commit: 77844e2591235bfc1944e901922f876c1d43c264
Parents: dca4f85
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Jun 16 13:42:41 2016 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Jun 16 13:45:29 2016 -0400
----------------------------------------------------------------------
.../org/apache/lucene/search/BooleanScorer.java | 6 +-
.../lucene/search/FilterLeafCollector.java | 7 +-
.../org/apache/lucene/search/TestBoolean2.java | 130 +++++++++++++++---
.../lucene/search/TestSimpleExplanations.java | 27 +++-
.../TestSimpleExplanationsWithFillerDocs.java | 126 ++++++++++++++++++
.../lucene/search/BaseExplanationTestCase.java | 43 ++++--
.../search/TestBaseExplanationTestCase.java | 131 +++++++++++++++++++
7 files changed, 428 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77844e25/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
index 4534bd4..73880a8 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
@@ -275,13 +275,13 @@ final class BooleanScorer extends BulkScorer {
}
}
- private void scoreWindowSingleScorer(BulkScorerAndDoc bulkScorer, LeafCollector collector,
+ private void scoreWindowSingleScorer(BulkScorerAndDoc bulkScorer, LeafCollector collector, LeafCollector singleClauseCollector,
Bits acceptDocs, int windowMin, int windowMax, int max) throws IOException {
assert tail.size() == 0;
final int nextWindowBase = head.top().next & ~MASK;
final int end = Math.max(windowMax, Math.min(max, nextWindowBase));
- bulkScorer.score(collector, acceptDocs, windowMin, end);
+ bulkScorer.score(singleClauseCollector, acceptDocs, windowMin, end);
// reset the scorer that should be used for the general case
collector.setScorer(fakeScorer);
@@ -304,7 +304,7 @@ final class BooleanScorer extends BulkScorer {
// special case: only one scorer can match in the current window,
// we can collect directly
final BulkScorerAndDoc bulkScorer = leads[0];
- scoreWindowSingleScorer(bulkScorer, singleClauseCollector, acceptDocs, windowMin, windowMax, max);
+ scoreWindowSingleScorer(bulkScorer, collector, singleClauseCollector, acceptDocs, windowMin, windowMax, max);
return head.add(bulkScorer);
} else {
// general case, collect through a bit set first and then replay
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77844e25/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java b/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java
index ab15bab..4b6ddfb 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java
@@ -45,7 +45,12 @@ public class FilterLeafCollector implements LeafCollector {
@Override
public String toString() {
- return getClass().getSimpleName() + "(" + in + ")";
+ String name = getClass().getSimpleName();
+ if (name.length() == 0) {
+ // an anonoymous subclass will have empty name?
+ name = "FilterLeafCollector";
+ }
+ return name + "(" + in + ")";
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77844e25/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
index f35f35d..d97d8d4 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2.java
@@ -18,15 +18,18 @@ package org.apache.lucene.search;
+import java.util.Arrays;
import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
@@ -42,24 +45,45 @@ import org.junit.Test;
*/
public class TestBoolean2 extends LuceneTestCase {
private static IndexSearcher searcher;
+ private static IndexSearcher singleSegmentSearcher;
private static IndexSearcher bigSearcher;
private static IndexReader reader;
private static IndexReader littleReader;
- private static int NUM_EXTRA_DOCS = 6000;
+ private static IndexReader singleSegmentReader;
+ /** num of empty docs injected between every doc in the (main) index */
+ private static int NUM_FILLER_DOCS;
+ /** num of empty docs injected prior to the first doc in the (main) index */
+ private static int PRE_FILLER_DOCS;
+ /** num "extra" docs containing value in "field2" added to the "big" clone of the index */
+ private static final int NUM_EXTRA_DOCS = 6000;
public static final String field = "field";
private static Directory directory;
+ private static Directory singleSegmentDirectory;
private static Directory dir2;
private static int mulFactor;
@BeforeClass
public static void beforeClass() throws Exception {
+ // in some runs, test immediate adjacency of matches - in others, force a full bucket gap betwen docs
+ NUM_FILLER_DOCS = random().nextBoolean() ? 0 : BooleanScorer.SIZE;
+ PRE_FILLER_DOCS = TestUtil.nextInt(random(), 0, (NUM_FILLER_DOCS / 2));
+
directory = newDirectory();
RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
+
+ Document doc = new Document();
+ for (int filler = 0; filler < PRE_FILLER_DOCS; filler++) {
+ writer.addDocument(doc);
+ }
for (int i = 0; i < docFields.length; i++) {
- Document doc = new Document();
doc.add(newTextField(field, docFields[i], Field.Store.NO));
writer.addDocument(doc);
+
+ doc = new Document();
+ for (int filler = 0; filler < NUM_FILLER_DOCS; filler++) {
+ writer.addDocument(doc);
+ }
}
writer.close();
littleReader = DirectoryReader.open(directory);
@@ -67,6 +91,18 @@ public class TestBoolean2 extends LuceneTestCase {
// this is intentionally using the baseline sim, because it compares against bigSearcher (which uses a random one)
searcher.setSimilarity(new DefaultSimilarity());
+ // make a copy of our index using a single segment
+ singleSegmentDirectory = new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(directory));
+ IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
+ // we need docID order to be preserved:
+ iwc.setMergePolicy(newLogMergePolicy());
+ try (IndexWriter w = new IndexWriter(singleSegmentDirectory, iwc)) {
+ w.forceMerge(1, true);
+ }
+ singleSegmentReader = DirectoryReader.open(singleSegmentDirectory);
+ singleSegmentSearcher = newSearcher(singleSegmentReader);
+ singleSegmentSearcher.setSimilarity(searcher.getSimilarity(true));
+
// Make big index
dir2 = new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(directory));
@@ -86,12 +122,12 @@ public class TestBoolean2 extends LuceneTestCase {
docCount = w.maxDoc();
w.close();
mulFactor *= 2;
- } while(docCount < 3000);
+ } while(docCount < 3000 * NUM_FILLER_DOCS);
RandomIndexWriter w = new RandomIndexWriter(random(), dir2,
newIndexWriterConfig(new MockAnalyzer(random()))
.setMaxBufferedDocs(TestUtil.nextInt(random(), 50, 1000)));
- Document doc = new Document();
+ doc = new Document();
doc.add(newTextField("field2", "xxx", Field.Store.NO));
for(int i=0;i<NUM_EXTRA_DOCS/2;i++) {
w.addDocument(doc);
@@ -110,8 +146,13 @@ public class TestBoolean2 extends LuceneTestCase {
public static void afterClass() throws Exception {
reader.close();
littleReader.close();
+ singleSegmentReader.close();
dir2.close();
directory.close();
+ singleSegmentDirectory.close();
+ singleSegmentSearcher = null;
+ singleSegmentReader = null;
+ singleSegmentDirectory = null;
searcher = null;
reader = null;
littleReader = null;
@@ -128,26 +169,57 @@ public class TestBoolean2 extends LuceneTestCase {
};
public void queriesTest(Query query, int[] expDocNrs) throws Exception {
+
+ // adjust the expected doc numbers according to our filler docs
+ if (0 < NUM_FILLER_DOCS) {
+ expDocNrs = Arrays.copyOf(expDocNrs, expDocNrs.length);
+ for (int i=0; i < expDocNrs.length; i++) {
+ expDocNrs[i] = PRE_FILLER_DOCS + ((NUM_FILLER_DOCS + 1) * expDocNrs[i]);
+ }
+ }
+
+ final int topDocsToCheck = atLeast(1000);
// The asserting searcher will sometimes return the bulk scorer and
// sometimes return a default impl around the scorer so that we can
// compare BS1 and BS2
- TopScoreDocCollector collector = TopScoreDocCollector.create(1000);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(topDocsToCheck);
searcher.search(query, collector);
ScoreDoc[] hits1 = collector.topDocs().scoreDocs;
-
- collector = TopScoreDocCollector.create(1000);
+ collector = TopScoreDocCollector.create(topDocsToCheck);
searcher.search(query, collector);
ScoreDoc[] hits2 = collector.topDocs().scoreDocs;
+ CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs);
+
+ // Since we have no deleted docs, we should also be able to verify identical matches &
+ // scores against an single segment copy of our index
+ collector = TopScoreDocCollector.create(topDocsToCheck);
+ singleSegmentSearcher.search(query, collector);
+ hits2 = collector.topDocs().scoreDocs;
+ CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs);
+
+ // sanity check expected num matches in bigSearcher
assertEquals(mulFactor * collector.totalHits,
bigSearcher.search(query, 1).totalHits);
-
- CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs);
+
+ // now check 2 diff scorers from the bigSearcher as well
+ collector = TopScoreDocCollector.create(topDocsToCheck);
+ bigSearcher.search(query, collector);
+ hits1 = collector.topDocs().scoreDocs;
+ collector = TopScoreDocCollector.create(topDocsToCheck);
+ bigSearcher.search(query, collector);
+ hits2 = collector.topDocs().scoreDocs;
+
+ // NOTE: just comparing results, not vetting against expDocNrs
+ // since we have dups in bigSearcher
+ CheckHits.checkEqual(query, hits1, hits2);
+
}
@Test
public void testQueries01() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST);
int[] expDocNrs = {2,3};
@@ -157,6 +229,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries02() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.SHOULD);
int[] expDocNrs = {2,3,1,0};
@@ -166,6 +239,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries03() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.SHOULD);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.SHOULD);
int[] expDocNrs = {2,3,1,0};
@@ -175,6 +249,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries04() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.SHOULD);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST_NOT);
int[] expDocNrs = {1,0};
@@ -184,6 +259,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries05() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST_NOT);
int[] expDocNrs = {1,0};
@@ -193,6 +269,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries06() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST_NOT);
query.add(new TermQuery(new Term(field, "w5")), BooleanClause.Occur.MUST_NOT);
@@ -203,6 +280,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries07() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST_NOT);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST_NOT);
query.add(new TermQuery(new Term(field, "w5")), BooleanClause.Occur.MUST_NOT);
@@ -213,6 +291,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries08() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.SHOULD);
query.add(new TermQuery(new Term(field, "w5")), BooleanClause.Occur.MUST_NOT);
@@ -223,6 +302,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries09() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "w2")), BooleanClause.Occur.MUST);
@@ -234,6 +314,7 @@ public class TestBoolean2 extends LuceneTestCase {
@Test
public void testQueries10() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(field, "w3")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "xx")), BooleanClause.Occur.MUST);
query.add(new TermQuery(new Term(field, "w2")), BooleanClause.Occur.MUST);
@@ -241,16 +322,19 @@ public class TestBoolean2 extends LuceneTestCase {
int[] expDocNrs = {2, 3};
Similarity oldSimilarity = searcher.getSimilarity(true);
- try {
- searcher.setSimilarity(new DefaultSimilarity(){
+ Similarity newSimilarity = new DefaultSimilarity() {
@Override
public float coord(int overlap, int maxOverlap) {
return overlap / ((float)maxOverlap - 1);
}
- });
+ };
+ try {
+ searcher.setSimilarity(newSimilarity);
+ singleSegmentSearcher.setSimilarity(newSimilarity);
queriesTest(query.build(), expDocNrs);
} finally {
searcher.setSimilarity(oldSimilarity);
+ singleSegmentSearcher.setSimilarity(oldSimilarity);
}
}
@@ -282,15 +366,11 @@ public class TestBoolean2 extends LuceneTestCase {
searcher.setSimilarity(new DefaultSimilarity()); // restore
}
- TopFieldCollector collector = TopFieldCollector.create(sort, 1000,
- false, true, true);
-
+ // check diff (randomized) scorers (from AssertingSearcher) produce the same results
+ TopFieldCollector collector = TopFieldCollector.create(sort, 1000, false, true, true);
searcher.search(q1, collector);
ScoreDoc[] hits1 = collector.topDocs().scoreDocs;
-
- collector = TopFieldCollector.create(sort, 1000,
- false, true, true);
-
+ collector = TopFieldCollector.create(sort, 1000, false, true, true);
searcher.search(q1, collector);
ScoreDoc[] hits2 = collector.topDocs().scoreDocs;
tot+=hits2.length;
@@ -301,6 +381,16 @@ public class TestBoolean2 extends LuceneTestCase {
q3.add(new PrefixQuery(new Term("field2", "b")), BooleanClause.Occur.SHOULD);
TopDocs hits4 = bigSearcher.search(q3.build(), 1);
assertEquals(mulFactor*collector.totalHits + NUM_EXTRA_DOCS/2, hits4.totalHits);
+
+ // test diff (randomized) scorers produce the same results on bigSearcher as well
+ collector = TopFieldCollector.create(sort, 1000 * mulFactor, false, true, true);
+ bigSearcher.search(q1, collector);
+ hits1 = collector.topDocs().scoreDocs;
+ collector = TopFieldCollector.create(sort, 1000 * mulFactor, false, true, true);
+ bigSearcher.search(q1, collector);
+ hits2 = collector.topDocs().scoreDocs;
+ CheckHits.checkEqual(q1, hits1, hits2);
+
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77844e25/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java b/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java
index d07c21e..0b48a02 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java
@@ -326,6 +326,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.MUST);
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
+ innerQuery.setDisableCoord(random().nextBoolean());
innerQuery.add(new TermQuery(new Term(FIELD, "qq")), BooleanClause.Occur.SHOULD);
BooleanQuery.Builder childLeft = new BooleanQuery.Builder();
@@ -347,6 +348,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.MUST);
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
+ innerQuery.setDisableCoord(random().nextBoolean());
innerQuery.add(new TermQuery(new Term(FIELD, "qq")), BooleanClause.Occur.SHOULD);
BooleanQuery.Builder childLeft = new BooleanQuery.Builder();
@@ -368,6 +370,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.MUST);
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
+ innerQuery.setDisableCoord(random().nextBoolean());
innerQuery.add(new TermQuery(new Term(FIELD, "qq")), BooleanClause.Occur.SHOULD);
BooleanQuery.Builder childLeft = new BooleanQuery.Builder();
@@ -389,6 +392,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.MUST);
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
+ innerQuery.setDisableCoord(random().nextBoolean());
innerQuery.add(new TermQuery(new Term(FIELD, "qq")), BooleanClause.Occur.SHOULD);
BooleanQuery.Builder childLeft = new BooleanQuery.Builder();
@@ -407,6 +411,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
}
public void testBQ11() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
TermQuery boostedQuery = new TermQuery(new Term(FIELD, "w1"));
query.add(new BoostQuery(boostedQuery, 1000), BooleanClause.Occur.SHOULD);
@@ -415,21 +420,21 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
}
public void testBQ14() throws Exception {
BooleanQuery.Builder q = new BooleanQuery.Builder();
- q.setDisableCoord(true);
+ q.setDisableCoord(random().nextBoolean());
q.add(new TermQuery(new Term(FIELD, "QQQQQ")), BooleanClause.Occur.SHOULD);
q.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
qtest(q.build(), new int[] { 0,1,2,3 });
}
public void testBQ15() throws Exception {
BooleanQuery.Builder q = new BooleanQuery.Builder();
- q.setDisableCoord(true);
+ q.setDisableCoord(random().nextBoolean());
q.add(new TermQuery(new Term(FIELD, "QQQQQ")), BooleanClause.Occur.MUST_NOT);
q.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
qtest(q.build(), new int[] { 0,1,2,3 });
}
public void testBQ16() throws Exception {
BooleanQuery.Builder q = new BooleanQuery.Builder();
- q.setDisableCoord(true);
+ q.setDisableCoord(random().nextBoolean());
q.add(new TermQuery(new Term(FIELD, "QQQQQ")), BooleanClause.Occur.SHOULD);
BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
@@ -441,7 +446,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
}
public void testBQ17() throws Exception {
BooleanQuery.Builder q = new BooleanQuery.Builder();
- q.setDisableCoord(true);
+ q.setDisableCoord(random().nextBoolean());
q.add(new TermQuery(new Term(FIELD, "w2")), BooleanClause.Occur.SHOULD);
BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
@@ -461,6 +466,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
public void testBQ20() throws Exception {
BooleanQuery.Builder q = new BooleanQuery.Builder();
+ q.setDisableCoord(random().nextBoolean());
q.setMinimumNumberShouldMatch(2);
q.add(new TermQuery(new Term(FIELD, "QQQQQ")), BooleanClause.Occur.SHOULD);
q.add(new TermQuery(new Term(FIELD, "yy")), BooleanClause.Occur.SHOULD);
@@ -471,6 +477,16 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
qtest(q.build(), new int[] { 0,3 });
}
+
+ public void testBQ21() throws Exception {
+ BooleanQuery.Builder q = new BooleanQuery.Builder();
+ q.setDisableCoord(random().nextBoolean());
+ q.add(new TermQuery(new Term(FIELD, "yy")), BooleanClause.Occur.SHOULD);
+ q.add(new TermQuery(new Term(FIELD, "zz")), BooleanClause.Occur.SHOULD);
+
+ qtest(q.build(), new int[] { 1,2,3 });
+
+ }
public void testBQ23() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
@@ -518,6 +534,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
}
public void testMultiFieldBQ3() throws Exception {
BooleanQuery.Builder query = new BooleanQuery.Builder();
+ query.setDisableCoord(random().nextBoolean());
query.add(new TermQuery(new Term(FIELD, "yy")), BooleanClause.Occur.SHOULD);
query.add(new TermQuery(new Term(ALTFIELD, "w3")), BooleanClause.Occur.MUST);
@@ -525,6 +542,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
}
public void testMultiFieldBQ4() throws Exception {
BooleanQuery.Builder outerQuery = new BooleanQuery.Builder();
+ outerQuery.setDisableCoord(random().nextBoolean());
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
@@ -536,6 +554,7 @@ public class TestSimpleExplanations extends BaseExplanationTestCase {
}
public void testMultiFieldBQ5() throws Exception {
BooleanQuery.Builder outerQuery = new BooleanQuery.Builder();
+ outerQuery.setDisableCoord(random().nextBoolean());
outerQuery.add(new TermQuery(new Term(FIELD, "w1")), BooleanClause.Occur.SHOULD);
BooleanQuery.Builder innerQuery = new BooleanQuery.Builder();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77844e25/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanationsWithFillerDocs.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanationsWithFillerDocs.java b/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanationsWithFillerDocs.java
new file mode 100644
index 0000000..0a79ae0
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanationsWithFillerDocs.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.util.Arrays;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.util.TestUtil;
+
+import org.junit.BeforeClass;
+import org.junit.Assume;
+
+
+/**
+ * subclass of TestSimpleExplanations that adds a lot of filler docs which will be ignored at query time.
+ * These filler docs will either all be empty in which case the queries will be unmodified, or they will
+ * all use terms from same set of source data as our regular docs (to emphasis the DocFreq factor in scoring),
+ * in which case the queries will be wrapped so they can be excluded.
+ */
+public class TestSimpleExplanationsWithFillerDocs extends TestSimpleExplanations {
+
+ /** num of empty docs injected between every doc in the index */
+ private static final int NUM_FILLER_DOCS = BooleanScorer.SIZE;
+ /** num of empty docs injected prior to the first doc in the (main) index */
+ private static int PRE_FILLER_DOCS;
+ /**
+ * If non-null then the filler docs are not empty, and need to be filtered out from queries
+ * using this as both field name & field value
+ */
+ public static String EXTRA = null;
+
+ private static final Document EMPTY_DOC = new Document();
+
+ /**
+ * Replaces the index created by our superclass with a new one that includes a lot of docs filler docs.
+ * {@link #qtest} will account for these extra filler docs.
+ * @see #qtest
+ */
+ @BeforeClass
+ public static void replaceIndex() throws Exception {
+ EXTRA = random().nextBoolean() ? null : "extra";
+ PRE_FILLER_DOCS = TestUtil.nextInt(random(), 0, (NUM_FILLER_DOCS / 2));
+
+ // free up what our super class created that we won't be using
+ reader.close();
+ directory.close();
+
+ directory = newDirectory();
+ try (RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy()))) {
+
+ for (int filler = 0; filler < PRE_FILLER_DOCS; filler++) {
+ writer.addDocument(makeFillerDoc());
+ }
+ for (int i = 0; i < docFields.length; i++) {
+ writer.addDocument(createDoc(i));
+
+ for (int filler = 0; filler < NUM_FILLER_DOCS; filler++) {
+ writer.addDocument(makeFillerDoc());
+ }
+ }
+ reader = writer.getReader();
+ searcher = newSearcher(reader);
+ }
+ }
+
+ private static Document makeFillerDoc() {
+ if (null == EXTRA) {
+ return EMPTY_DOC;
+ }
+ Document doc = createDoc(TestUtil.nextInt(random(), 0, docFields.length-1));
+ doc.add(newStringField(EXTRA, EXTRA, Field.Store.NO));
+ return doc;
+ }
+
+ /**
+ * Adjusts <code>expDocNrs</code> based on the filler docs injected in the index,
+ * and if neccessary wraps the <code>q</code> in a BooleanQuery that will filter out all
+ * filler docs using the {@link #EXTRA} field.
+ *
+ * @see #replaceIndex
+ */
+ @Override
+ public void qtest(Query q, int[] expDocNrs) throws Exception {
+
+ expDocNrs = Arrays.copyOf(expDocNrs, expDocNrs.length);
+ for (int i=0; i < expDocNrs.length; i++) {
+ expDocNrs[i] = PRE_FILLER_DOCS + ((NUM_FILLER_DOCS + 1) * expDocNrs[i]);
+ }
+
+ if (null != EXTRA) {
+ BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ builder.add(new BooleanClause(q, BooleanClause.Occur.MUST));
+ builder.add(new BooleanClause(new TermQuery(new Term(EXTRA, EXTRA)), BooleanClause.Occur.MUST_NOT));
+ q = builder.build();
+ }
+ super.qtest(q, expDocNrs);
+ }
+
+ public void testMA1() throws Exception {
+ Assume.assumeNotNull("test is not viable with empty filler docs", EXTRA);
+ super.testMA1();
+ }
+ public void testMA2() throws Exception {
+ Assume.assumeNotNull("test is not viable with empty filler docs", EXTRA);
+ super.testMA2();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77844e25/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
index cde2cdd..7775f11 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/BaseExplanationTestCase.java
@@ -71,22 +71,26 @@ public abstract class BaseExplanationTestCase extends LuceneTestCase {
public static void beforeClassTestExplanations() throws Exception {
directory = newDirectory();
analyzer = new MockAnalyzer(random());
- RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy()));
- for (int i = 0; i < docFields.length; i++) {
- Document doc = new Document();
- doc.add(newStringField(KEY, ""+i, Field.Store.NO));
- doc.add(new SortedDocValuesField(KEY, new BytesRef(""+i)));
- Field f = newTextField(FIELD, docFields[i], Field.Store.NO);
- f.setBoost(i);
- doc.add(f);
- doc.add(newTextField(ALTFIELD, docFields[i], Field.Store.NO));
- writer.addDocument(doc);
+ try (RandomIndexWriter writer = new RandomIndexWriter(random(), directory, newIndexWriterConfig(analyzer).setMergePolicy(newLogMergePolicy()))) {
+ for (int i = 0; i < docFields.length; i++) {
+ writer.addDocument(createDoc(i));
+ }
+ reader = writer.getReader();
+ searcher = newSearcher(reader);
}
- reader = writer.getReader();
- writer.close();
- searcher = newSearcher(reader);
}
+ public static Document createDoc(int index) {
+ Document doc = new Document();
+ doc.add(newStringField(KEY, ""+index, Field.Store.NO));
+ doc.add(new SortedDocValuesField(KEY, new BytesRef(""+index)));
+ Field f = newTextField(FIELD, docFields[index], Field.Store.NO);
+ f.setBoost(index);
+ doc.add(f);
+ doc.add(newTextField(ALTFIELD, docFields[index], Field.Store.NO));
+ return doc;
+ }
+
protected static final String[] docFields = {
"w1 w2 w3 w4 w5",
"w1 w3 w2 w3 zz",
@@ -94,8 +98,19 @@ public abstract class BaseExplanationTestCase extends LuceneTestCase {
"w1 w3 xx w2 yy w3 zz"
};
- /** check the expDocNrs first, then check the query (and the explanations) */
+ /**
+ * check the expDocNrs match and have scores that match the explanations.
+ * Query may be randomly wrapped in a BooleanQuery with a term that matches no documents in
+ * order to trigger coord logic.
+ */
public void qtest(Query q, int[] expDocNrs) throws Exception {
+ if (random().nextBoolean()) {
+ BooleanQuery.Builder bq = new BooleanQuery.Builder();
+ bq.setDisableCoord(random().nextBoolean());
+ bq.add(q, BooleanClause.Occur.SHOULD);
+ bq.add(new TermQuery(new Term("NEVER","MATCH")), BooleanClause.Occur.SHOULD);
+ q = bq.build();
+ }
CheckHits.checkHitCollector(random(), q, FIELD, searcher, expDocNrs);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/77844e25/lucene/test-framework/src/test/org/apache/lucene/search/TestBaseExplanationTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/test/org/apache/lucene/search/TestBaseExplanationTestCase.java b/lucene/test-framework/src/test/org/apache/lucene/search/TestBaseExplanationTestCase.java
new file mode 100644
index 0000000..59d7aeb
--- /dev/null
+++ b/lucene/test-framework/src/test/org/apache/lucene/search/TestBaseExplanationTestCase.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+
+
+import junit.framework.AssertionFailedError;
+
+/**
+ * Tests that the {@link BaseExplanationTestCase} helper code, as well as
+ * {@link CheckHits#checkNoMatchExplanations} are checking what they are suppose to.
+ */
+public class TestBaseExplanationTestCase extends BaseExplanationTestCase {
+
+ public void testQueryNoMatchWhenExpected() throws Exception {
+ try {
+ qtest(new TermQuery(new Term(FIELD, "BOGUS")), new int[]{3 /* none */});
+ fail("Expected an AssertionFailedError but there wasn't one");
+ } catch (AssertionFailedError expected) {
+ // drop the expected exception
+ }
+ }
+ public void testQueryMatchWhenNotExpected() throws Exception {
+ try {
+ qtest(new TermQuery(new Term(FIELD, "w1")), new int[] { 0, 1 /*, 2, 3 */ });
+ fail("Expected an AssertionFailedError but there wasn't one");
+ } catch (AssertionFailedError expected) {
+ // drop the expected exception
+ }
+ }
+
+ public void testIncorrectExplainScores() throws Exception {
+ // sanity check what a real TermQuery matches
+ qtest(new TermQuery(new Term(FIELD, "zz")), new int[] { 1, 3 });
+
+ // ensure when the Explanations are broken, we get an error about those matches
+ try {
+ qtest(new BrokenExplainTermQuery(new Term(FIELD, "zz"), false, true), new int[] { 1, 3 });
+ fail("Expected an AssertionFailedError but there wasn't one");
+ } catch (AssertionFailedError expected) {
+ // drop the expected exception
+ }
+ }
+
+ public void testIncorrectExplainMatches() throws Exception {
+ // sanity check what a real TermQuery matches
+ qtest(new TermQuery(new Term(FIELD, "zz")), new int[] { 1, 3 });
+
+ // ensure when the Explanations are broken, we get an error about the non matches
+ try {
+ CheckHits.checkNoMatchExplanations(new BrokenExplainTermQuery(new Term(FIELD, "zz"), true, false),
+ FIELD, searcher, new int[] { 1, 3 });
+ fail("Expected an AssertionFailedError but there wasn't one");
+ } catch (AssertionFailedError expected) {
+ // drop the expected exception
+ }
+ }
+
+
+ public static final class BrokenExplainTermQuery extends TermQuery {
+ public final boolean toggleExplainMatch;
+ public final boolean breakExplainScores;
+ public BrokenExplainTermQuery(Term t, boolean toggleExplainMatch, boolean breakExplainScores) {
+ super(t);
+ this.toggleExplainMatch = toggleExplainMatch;
+ this.breakExplainScores = breakExplainScores;
+ }
+ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+ return new BrokenExplainWeight(this, super.createWeight(searcher,needsScores));
+ }
+ }
+
+ public static final class BrokenExplainWeight extends Weight {
+ final Weight in;
+ public BrokenExplainWeight(BrokenExplainTermQuery q, Weight in) {
+ super(q);
+ this.in = in;
+ }
+ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
+ return in.bulkScorer(context);
+ }
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ BrokenExplainTermQuery q = (BrokenExplainTermQuery) this.getQuery();
+ Explanation result = in.explain(context, doc);
+ if (result.isMatch()) {
+ if (q.breakExplainScores) {
+ result = Explanation.match(-1F * result.getValue(), "Broken Explanation Score", result);
+ }
+ if (q.toggleExplainMatch) {
+ result = Explanation.noMatch("Broken Explanation Matching", result);
+ }
+ } else {
+ if (q.toggleExplainMatch) {
+ result = Explanation.match(-42.0F, "Broken Explanation Matching", result);
+ }
+ }
+ return result;
+ }
+ public void extractTerms(Set<Term> terms) {
+ in.extractTerms(terms);
+ }
+ public float getValueForNormalization() throws IOException {
+ return in.getValueForNormalization();
+ }
+ public void normalize(float norm, float boost) {
+ in.normalize(norm, boost);
+ }
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ return in.scorer(context);
+ }
+ }
+}