You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2019/09/13 15:44:37 UTC
[lucene-solr] 02/02: LUCENE-8939: Introduce Shared Count Early
Termination In Parallel Search (#823)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 55d9290433ea7f47fd722197b32811b26bf3ee8f
Author: Atri Sharma <at...@gmail.com>
AuthorDate: Thu Sep 5 13:37:59 2019 +0530
LUCENE-8939: Introduce Shared Count Early Termination In Parallel Search (#823)
This commit introduces a strategy to early terminate for sorted
collections during parallel search when requested number of
hits have been collected but the total hits threshold has not
yet been reached.
---
lucene/CHANGES.txt | 3 +
.../apache/lucene/search/HitsThresholdChecker.java | 119 +++++++++++++++++++++
.../org/apache/lucene/search/IndexSearcher.java | 9 +-
.../apache/lucene/search/TopFieldCollector.java | 79 ++++++++++----
.../apache/lucene/search/TopScoreDocCollector.java | 67 +++++++++---
.../apache/lucene/search/TestTopDocsCollector.java | 53 +++++++++
.../lucene/search/TestTopFieldCollector.java | 36 +++++++
7 files changed, 329 insertions(+), 37 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2e2130a..594ce10 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -94,6 +94,9 @@ the total hits is not requested.
* LUCENE-8968: Improve performance of WITHIN and DISJOINT queries for Shape queries by
doing just one pass whenever possible. (Ignacio Vera)
+* LUCENE-8939: Introduce shared count based early termination across multiple slices
+ (Atri Sharma)
+
Bug Fixes
* LUCENE-8755: spatial-extras quad and packed quad prefix trees could throw a
diff --git a/lucene/core/src/java/org/apache/lucene/search/HitsThresholdChecker.java b/lucene/core/src/java/org/apache/lucene/search/HitsThresholdChecker.java
new file mode 100644
index 0000000..5453660
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/HitsThresholdChecker.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Used for defining custom algorithms to allow searches to early terminate
+ */
+abstract class HitsThresholdChecker {
+ /**
+ * Implementation of HitsThresholdChecker which allows global hit counting
+ */
+ private static class GlobalHitsThresholdChecker extends HitsThresholdChecker {
+ private final int totalHitsThreshold;
+ private final AtomicLong globalHitCount;
+
+ public GlobalHitsThresholdChecker(int totalHitsThreshold) {
+
+ if (totalHitsThreshold < 0) {
+ throw new IllegalArgumentException("totalHitsThreshold must be >= 0, got " + totalHitsThreshold);
+ }
+
+ this.totalHitsThreshold = totalHitsThreshold;
+ this.globalHitCount = new AtomicLong();
+ }
+
+ @Override
+ public void incrementHitCount() {
+ globalHitCount.incrementAndGet();
+ }
+
+ @Override
+ public boolean isThresholdReached(){
+ return globalHitCount.get() > totalHitsThreshold;
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ return totalHitsThreshold == Integer.MAX_VALUE ? ScoreMode.COMPLETE : ScoreMode.TOP_SCORES;
+ }
+
+ @Override
+ public int getHitsThreshold() {
+ return totalHitsThreshold;
+ }
+ }
+
+ /**
+ * Default implementation of HitsThresholdChecker to be used for single threaded execution
+ */
+ private static class LocalHitsThresholdChecker extends HitsThresholdChecker {
+ private final int totalHitsThreshold;
+ private int hitCount;
+
+ public LocalHitsThresholdChecker(int totalHitsThreshold) {
+
+ if (totalHitsThreshold < 0) {
+ throw new IllegalArgumentException("totalHitsThreshold must be >= 0, got " + totalHitsThreshold);
+ }
+
+ this.totalHitsThreshold = totalHitsThreshold;
+ }
+
+ @Override
+ public void incrementHitCount() {
+ ++hitCount;
+ }
+
+ @Override
+ public boolean isThresholdReached() {
+ return hitCount > totalHitsThreshold;
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ return totalHitsThreshold == Integer.MAX_VALUE ? ScoreMode.COMPLETE : ScoreMode.TOP_SCORES;
+ }
+
+ @Override
+ public int getHitsThreshold() {
+ return totalHitsThreshold;
+ }
+ }
+
+ /*
+ * Returns a threshold checker that is useful for single threaded searches
+ */
+ public static HitsThresholdChecker create(final int totalHitsThreshold) {
+ return new LocalHitsThresholdChecker(totalHitsThreshold);
+ }
+
+ /*
+ * Returns a threshold checker that is based on a shared counter
+ */
+ public static HitsThresholdChecker createShared(final int totalHitsThreshold) {
+ return new GlobalHitsThresholdChecker(totalHitsThreshold);
+ }
+
+ public abstract void incrementHitCount();
+ public abstract ScoreMode scoreMode();
+ public abstract int getHitsThreshold();
+ public abstract boolean isThresholdReached();
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
index a6198fb..ba9972a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
@@ -396,9 +396,11 @@ public class IndexSearcher {
final CollectorManager<TopScoreDocCollector, TopDocs> manager = new CollectorManager<TopScoreDocCollector, TopDocs>() {
+ private final HitsThresholdChecker hitsThresholdChecker = (executor == null || leafSlices.length <= 1) ? HitsThresholdChecker.create(TOTAL_HITS_THRESHOLD) :
+ HitsThresholdChecker.createShared(TOTAL_HITS_THRESHOLD);
@Override
public TopScoreDocCollector newCollector() throws IOException {
- return TopScoreDocCollector.create(cappedNumHits, after, TOTAL_HITS_THRESHOLD);
+ return TopScoreDocCollector.create(cappedNumHits, after, hitsThresholdChecker);
}
@Override
@@ -524,10 +526,13 @@ public class IndexSearcher {
final CollectorManager<TopFieldCollector, TopFieldDocs> manager = new CollectorManager<TopFieldCollector, TopFieldDocs>() {
+ private final HitsThresholdChecker hitsThresholdChecker = (executor == null || leafSlices.length <= 1) ? HitsThresholdChecker.create(TOTAL_HITS_THRESHOLD) :
+ HitsThresholdChecker.createShared(TOTAL_HITS_THRESHOLD);
+
@Override
public TopFieldCollector newCollector() throws IOException {
// TODO: don't pay the price for accurate hit counts by default
- return TopFieldCollector.create(rewrittenSort, cappedNumHits, after, TOTAL_HITS_THRESHOLD);
+ return TopFieldCollector.create(rewrittenSort, cappedNumHits, after, hitsThresholdChecker);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
index 5981659..7a09b5b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Comparator;
import java.util.List;
@@ -96,12 +97,12 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
* document scores and maxScore.
*/
private static class SimpleFieldCollector extends TopFieldCollector {
-
final Sort sort;
final FieldValueHitQueue<Entry> queue;
- public SimpleFieldCollector(Sort sort, FieldValueHitQueue<Entry> queue, int numHits, int totalHitsThreshold) {
- super(queue, numHits, totalHitsThreshold, sort.needsScores());
+ public SimpleFieldCollector(Sort sort, FieldValueHitQueue<Entry> queue, int numHits,
+ HitsThresholdChecker hitsThresholdChecker) {
+ super(queue, numHits, hitsThresholdChecker, sort.needsScores());
this.sort = sort;
this.queue = queue;
}
@@ -128,13 +129,14 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
@Override
public void collect(int doc) throws IOException {
++totalHits;
+ hitsThresholdChecker.incrementHitCount();
if (queueFull) {
if (collectedAllCompetitiveHits || reverseMul * comparator.compareBottom(doc) <= 0) {
// since docs are visited in doc Id order, if compare is 0, it means
// this document is largest than anything else in the queue, and
// therefore not competitive.
if (canEarlyTerminate) {
- if (totalHits > totalHitsThreshold) {
+ if (hitsThresholdChecker.isThresholdReached()) {
totalHitsRelation = Relation.GREATER_THAN_OR_EQUAL_TO;
throw new CollectionTerminatedException();
} else {
@@ -181,15 +183,13 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
int collectedHits;
final FieldValueHitQueue<Entry> queue;
final FieldDoc after;
- final int totalHitsThreshold;
public PagingFieldCollector(Sort sort, FieldValueHitQueue<Entry> queue, FieldDoc after, int numHits,
- int totalHitsThreshold) {
- super(queue, numHits, totalHitsThreshold, sort.needsScores());
+ HitsThresholdChecker hitsThresholdChecker) {
+ super(queue, numHits, hitsThresholdChecker, sort.needsScores());
this.sort = sort;
this.queue = queue;
this.after = after;
- this.totalHitsThreshold = totalHitsThreshold;
FieldComparator<?>[] comparators = queue.comparators;
// Tell all comparators their top value:
@@ -221,6 +221,7 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
//System.out.println(" collect doc=" + doc);
totalHits++;
+ hitsThresholdChecker.incrementHitCount();
if (queueFull) {
// Fastmatch: return if this hit is no better than
@@ -230,7 +231,7 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
// this document is largest than anything else in the queue, and
// therefore not competitive.
if (canEarlyTerminate) {
- if (totalHits > totalHitsThreshold) {
+ if (hitsThresholdChecker.isThresholdReached()) {
totalHitsRelation = Relation.GREATER_THAN_OR_EQUAL_TO;
throw new CollectionTerminatedException();
} else {
@@ -282,7 +283,7 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
private static final ScoreDoc[] EMPTY_SCOREDOCS = new ScoreDoc[0];
final int numHits;
- final int totalHitsThreshold;
+ final HitsThresholdChecker hitsThresholdChecker;
final FieldComparator.RelevanceComparator firstComparator;
final boolean canSetMinScore;
final int numComparators;
@@ -297,17 +298,18 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
// internal versions. If someone will define a constructor with any other
// visibility, then anyone will be able to extend the class, which is not what
// we want.
- private TopFieldCollector(FieldValueHitQueue<Entry> pq, int numHits, int totalHitsThreshold, boolean needsScores) {
+ private TopFieldCollector(FieldValueHitQueue<Entry> pq, int numHits,
+ HitsThresholdChecker hitsThresholdChecker, boolean needsScores) {
super(pq);
this.needsScores = needsScores;
this.numHits = numHits;
- this.totalHitsThreshold = totalHitsThreshold;
+ this.hitsThresholdChecker = hitsThresholdChecker;
this.numComparators = pq.getComparators().length;
FieldComparator<?> fieldComparator = pq.getComparators()[0];
int reverseMul = pq.reverseMul[0];
if (fieldComparator.getClass().equals(FieldComparator.RelevanceComparator.class)
&& reverseMul == 1 // if the natural sort is preserved (sort by descending relevance)
- && totalHitsThreshold != Integer.MAX_VALUE) {
+ && hitsThresholdChecker.getHitsThreshold() != Integer.MAX_VALUE) {
firstComparator = (FieldComparator.RelevanceComparator) fieldComparator;
scoreMode = ScoreMode.TOP_SCORES;
canSetMinScore = true;
@@ -324,7 +326,7 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
}
protected void updateMinCompetitiveScore(Scorable scorer) throws IOException {
- if (canSetMinScore && totalHits > totalHitsThreshold && queueFull) {
+ if (canSetMinScore && hitsThresholdChecker.isThresholdReached() && queueFull) {
assert bottom != null && firstComparator != null;
float minScore = firstComparator.value(bottom.slot);
scorer.setMinCompetitiveScore(minScore);
@@ -382,8 +384,19 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
* @return a {@link TopFieldCollector} instance which will sort the results by
* the sort criteria.
*/
- public static TopFieldCollector create(Sort sort, int numHits, FieldDoc after,
- int totalHitsThreshold) {
+ public static TopFieldCollector create(Sort sort, int numHits, FieldDoc after, int totalHitsThreshold) {
+ if (totalHitsThreshold < 0) {
+ throw new IllegalArgumentException("totalHitsThreshold must be >= 0, got " + totalHitsThreshold);
+ }
+
+ return create(sort, numHits, after, HitsThresholdChecker.create(totalHitsThreshold));
+ }
+
+ /**
+ * Same as above with an additional parameter to allow passing in the threshold checker
+ */
+ static TopFieldCollector create(Sort sort, int numHits, FieldDoc after,
+ HitsThresholdChecker hitsThresholdChecker) {
if (sort.fields.length == 0) {
throw new IllegalArgumentException("Sort must contain at least one field");
@@ -393,14 +406,14 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
throw new IllegalArgumentException("numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count");
}
- if (totalHitsThreshold < 0) {
- throw new IllegalArgumentException("totalHitsThreshold must be >= 0, got " + totalHitsThreshold);
+ if (hitsThresholdChecker == null) {
+ throw new IllegalArgumentException("hitsThresholdChecker should not be null");
}
FieldValueHitQueue<Entry> queue = FieldValueHitQueue.create(sort.fields, numHits);
if (after == null) {
- return new SimpleFieldCollector(sort, queue, numHits, totalHitsThreshold);
+ return new SimpleFieldCollector(sort, queue, numHits, hitsThresholdChecker);
} else {
if (after.fields == null) {
throw new IllegalArgumentException("after.fields wasn't set; you must pass fillFields=true for the previous search");
@@ -410,11 +423,37 @@ public abstract class TopFieldCollector extends TopDocsCollector<Entry> {
throw new IllegalArgumentException("after.fields has " + after.fields.length + " values but sort has " + sort.getSort().length);
}
- return new PagingFieldCollector(sort, queue, after, numHits, totalHitsThreshold);
+ return new PagingFieldCollector(sort, queue, after, numHits, hitsThresholdChecker);
}
}
/**
+ * Create a CollectorManager which uses a shared hit counter to maintain number of hits
+ */
+ public static CollectorManager<TopFieldCollector, TopFieldDocs> createSharedManager(Sort sort, int numHits, FieldDoc after,
+ int totalHitsThreshold) {
+ return new CollectorManager<TopFieldCollector, TopFieldDocs>() {
+
+ private final HitsThresholdChecker hitsThresholdChecker = HitsThresholdChecker.createShared(totalHitsThreshold);
+
+ @Override
+ public TopFieldCollector newCollector() throws IOException {
+ return create(sort, numHits, after, hitsThresholdChecker);
+ }
+
+ @Override
+ public TopFieldDocs reduce(Collection<TopFieldCollector> collectors) throws IOException {
+ final TopFieldDocs[] topDocs = new TopFieldDocs[collectors.size()];
+ int i = 0;
+ for (TopFieldCollector collector : collectors) {
+ topDocs[i++] = collector.topDocs();
+ }
+ return TopDocs.merge(sort, numHits, topDocs);
+ }
+ };
+ }
+
+ /**
* Populate {@link ScoreDoc#score scores} of the given {@code topDocs}.
* @param topDocs the top docs to populate
* @param searcher the index searcher that has been used to compute {@code topDocs}
diff --git a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java
index c857f8f..b0e6a8b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TopScoreDocCollector.java
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
import java.io.IOException;
+import java.util.Collection;
import org.apache.lucene.index.LeafReaderContext;
@@ -48,8 +49,8 @@ public abstract class TopScoreDocCollector extends TopDocsCollector<ScoreDoc> {
private static class SimpleTopScoreDocCollector extends TopScoreDocCollector {
- SimpleTopScoreDocCollector(int numHits, int totalHitsThreshold) {
- super(numHits, totalHitsThreshold);
+ SimpleTopScoreDocCollector(int numHits, HitsThresholdChecker hitsThresholdChecker) {
+ super(numHits, hitsThresholdChecker);
}
@Override
@@ -71,8 +72,10 @@ public abstract class TopScoreDocCollector extends TopDocsCollector<ScoreDoc> {
assert score >= 0; // NOTE: false for NaN
totalHits++;
+ hitsThresholdChecker.incrementHitCount();
+
if (score <= pqTop.score) {
- if (totalHitsRelation == TotalHits.Relation.EQUAL_TO && totalHits > totalHitsThreshold) {
+ if (totalHitsRelation == TotalHits.Relation.EQUAL_TO && hitsThresholdChecker.isThresholdReached()) {
// we just reached totalHitsThreshold, we can start setting the min
// competitive score now
updateMinCompetitiveScore(scorer);
@@ -97,8 +100,8 @@ public abstract class TopScoreDocCollector extends TopDocsCollector<ScoreDoc> {
private final ScoreDoc after;
private int collectedHits;
- PagingTopScoreDocCollector(int numHits, ScoreDoc after, int totalHitsThreshold) {
- super(numHits, totalHitsThreshold);
+ PagingTopScoreDocCollector(int numHits, ScoreDoc after, HitsThresholdChecker hitsThresholdChecker) {
+ super(numHits, hitsThresholdChecker);
this.after = after;
this.collectedHits = 0;
}
@@ -130,10 +133,11 @@ public abstract class TopScoreDocCollector extends TopDocsCollector<ScoreDoc> {
assert score >= 0; // NOTE: false for NaN
totalHits++;
+ hitsThresholdChecker.incrementHitCount();
if (score > after.score || (score == after.score && doc <= afterDoc)) {
// hit was collected on a previous page
- if (totalHitsRelation == TotalHits.Relation.EQUAL_TO && totalHits > totalHitsThreshold) {
+ if (totalHitsRelation == TotalHits.Relation.EQUAL_TO && hitsThresholdChecker.isThresholdReached()) {
// we just reached totalHitsThreshold, we can start setting the min
// competitive score now
updateMinCompetitiveScore(scorer);
@@ -191,32 +195,65 @@ public abstract class TopScoreDocCollector extends TopDocsCollector<ScoreDoc> {
* objects.
*/
public static TopScoreDocCollector create(int numHits, ScoreDoc after, int totalHitsThreshold) {
+ return create(numHits, after, HitsThresholdChecker.create(totalHitsThreshold));
+ }
+
+ static TopScoreDocCollector create(int numHits, ScoreDoc after, HitsThresholdChecker hitsThresholdChecker) {
if (numHits <= 0) {
throw new IllegalArgumentException("numHits must be > 0; please use TotalHitCountCollector if you just need the total hit count");
}
- if (totalHitsThreshold < 0) {
- throw new IllegalArgumentException("totalHitsThreshold must be >= 0, got " + totalHitsThreshold);
+ if (hitsThresholdChecker == null) {
+ throw new IllegalArgumentException("hitsThresholdChecker must be non null");
}
if (after == null) {
- return new SimpleTopScoreDocCollector(numHits, totalHitsThreshold);
+ return new SimpleTopScoreDocCollector(numHits, hitsThresholdChecker);
} else {
- return new PagingTopScoreDocCollector(numHits, after, totalHitsThreshold);
+ return new PagingTopScoreDocCollector(numHits, after, hitsThresholdChecker);
}
}
- final int totalHitsThreshold;
+ /**
+ * Create a CollectorManager which uses a shared hit counter to maintain number of hits
+ */
+ public static CollectorManager<TopScoreDocCollector, TopDocs> createSharedManager(int numHits, FieldDoc after,
+ int totalHitsThreshold) {
+ return new CollectorManager<TopScoreDocCollector, TopDocs>() {
+
+ private final HitsThresholdChecker hitsThresholdChecker = HitsThresholdChecker.createShared(totalHitsThreshold);
+
+ @Override
+ public TopScoreDocCollector newCollector() throws IOException {
+ return TopScoreDocCollector.create(numHits, after, hitsThresholdChecker);
+ }
+
+ @Override
+ public TopDocs reduce(Collection<TopScoreDocCollector> collectors) throws IOException {
+ final TopDocs[] topDocs = new TopDocs[collectors.size()];
+ int i = 0;
+ for (TopScoreDocCollector collector : collectors) {
+ topDocs[i++] = collector.topDocs();
+ }
+ return TopDocs.merge(numHits, topDocs);
+ }
+
+ };
+ }
+
ScoreDoc pqTop;
+ final HitsThresholdChecker hitsThresholdChecker;
// prevents instantiation
- TopScoreDocCollector(int numHits, int totalHitsThreshold) {
+ TopScoreDocCollector(int numHits, HitsThresholdChecker hitsThresholdChecker) {
super(new HitQueue(numHits, true));
- this.totalHitsThreshold = totalHitsThreshold;
+ assert hitsThresholdChecker != null;
+
// HitQueue implements getSentinelObject to return a ScoreDoc, so we know
// that at this point top() is already initialized.
pqTop = pq.top();
+ this.hitsThresholdChecker = hitsThresholdChecker;
}
@Override
@@ -230,11 +267,11 @@ public abstract class TopScoreDocCollector extends TopDocsCollector<ScoreDoc> {
@Override
public ScoreMode scoreMode() {
- return totalHitsThreshold == Integer.MAX_VALUE ? ScoreMode.COMPLETE : ScoreMode.TOP_SCORES;
+ return hitsThresholdChecker.scoreMode();
}
protected void updateMinCompetitiveScore(Scorable scorer) throws IOException {
- if (totalHits > totalHitsThreshold
+ if (hitsThresholdChecker.isThresholdReached()
&& pqTop != null
&& pqTop.score != Float.NEGATIVE_INFINITY) { // -Infinity is the score of sentinels
// since we tie-break on doc id and collect in doc id order, we can require
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java
index 809d835..608108c 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java
@@ -19,6 +19,10 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
@@ -29,6 +33,7 @@ import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NamedThreadFactory;
public class TestTopDocsCollector extends LuceneTestCase {
@@ -96,6 +101,31 @@ public class TestTopDocsCollector extends LuceneTestCase {
searcher.search(q, tdc);
return tdc;
}
+
+ private TopDocsCollector<ScoreDoc> doSearchWithThreshold(int numResults, int thresHold) throws IOException {
+ Query q = new MatchAllDocsQuery();
+ IndexSearcher searcher = newSearcher(reader);
+ TopDocsCollector<ScoreDoc> tdc = TopScoreDocCollector.create(numResults, thresHold);
+ searcher.search(q, tdc);
+ return tdc;
+ }
+
+ private TopDocs doConcurrentSearchWithThreshold(int numResults, int thresHold) throws IOException {
+ Query q = new MatchAllDocsQuery();
+ ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(),
+ new NamedThreadFactory("TestTopDocsCollector"));
+ IndexSearcher searcher = new IndexSearcher(reader, service);
+
+ CollectorManager collectorManager = TopScoreDocCollector.createSharedManager(numResults,
+ null, Integer.MAX_VALUE);
+
+ TopDocs tdc = (TopDocs) searcher.search(q, collectorManager);
+
+ service.shutdown();
+
+ return tdc;
+ }
@Override
public void setUp() throws Exception {
@@ -274,6 +304,29 @@ public class TestTopDocsCollector extends LuceneTestCase {
dir.close();
}
+ public void testSharedCountCollectorManager() throws Exception {
+ Query q = new MatchAllDocsQuery();
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
+ Document doc = new Document();
+ w.addDocuments(Arrays.asList(doc, doc, doc, doc));
+ w.flush();
+ w.addDocuments(Arrays.asList(doc, doc));
+ w.flush();
+ IndexReader reader = DirectoryReader.open(w);
+ assertEquals(2, reader.leaves().size());
+ w.close();
+
+ TopDocsCollector collector = doSearchWithThreshold(5, 10);
+ TopDocs tdc = doConcurrentSearchWithThreshold(5, 10);
+ TopDocs tdc2 = collector.topDocs();
+
+ CheckHits.checkEqual(q, tdc.scoreDocs, tdc2.scoreDocs);
+
+ reader.close();
+ dir.close();
+ }
+
public void testTotalHits() throws Exception {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
index 5846733..2f5599d 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
@@ -20,6 +20,10 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
@@ -37,6 +41,7 @@ import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.FieldValueHitQueue.Entry;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.TestUtil;
import static org.apache.lucene.search.SortField.FIELD_SCORE;
@@ -108,6 +113,37 @@ public class TestTopFieldCollector extends LuceneTestCase {
}
}
+ public void testSharedHitcountCollector() throws Exception {
+
+ ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(),
+ new NamedThreadFactory("TestTopFieldCollector"));
+
+ IndexSearcher concurrentSearcher = new IndexSearcher(ir, service);
+
+ // Two Sort criteria to instantiate the multi/single comparators.
+ Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
+ for(int i = 0; i < sort.length; i++) {
+ Query q = new MatchAllDocsQuery();
+ TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, Integer.MAX_VALUE);
+
+ is.search(q, tdc);
+
+ CollectorManager tsdc = TopFieldCollector.createSharedManager(sort[i], 10, null, Integer.MAX_VALUE);
+
+ TopDocs td = tdc.topDocs();
+ TopDocs td2 = (TopDocs) concurrentSearcher.search(q, tsdc);
+ ScoreDoc[] sd = td.scoreDocs;
+ for(int j = 0; j < sd.length; j++) {
+ assertTrue(Float.isNaN(sd[j].score));
+ }
+
+ CheckHits.checkEqual(q, td.scoreDocs, td2.scoreDocs);
+ }
+
+ service.shutdown();
+ }
+
public void testSortWithoutTotalHitTracking() throws Exception {
Sort sort = new Sort(SortField.FIELD_DOC);
for(int i = 0; i < 2; i++) {