You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2015/02/27 18:37:32 UTC
svn commit: r1662761 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/search/
Author: jpountz
Date: Fri Feb 27 17:37:31 2015
New Revision: 1662761
URL: http://svn.apache.org/r1662761
Log:
LUCENE-6294: Generalize how IndexSearcher parallelizes collection execution.
Added:
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CollectorManager.java
- copied unchanged from r1662751, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CollectorManager.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1662761&r1=1662760&r2=1662761&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Fri Feb 27 17:37:31 2015
@@ -16,6 +16,9 @@ New Features
* LUCENE-6227: Added BooleanClause.Occur.FILTER to filter documents without
participating in scoring (on the contrary to MUST). (Adrien Grand)
+* LUCENE-6294: Added oal.search.CollectorManager to allow for parallelization
+ of the document collection process on IndexSearcher. (Adrien Grand)
+
Bug Fixes
* LUCENE-6249: StandardQueryParser doesn't support pure negative clauses.
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1662761&r1=1662760&r2=1662761&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java Fri Feb 27 17:37:31 2015
@@ -20,6 +20,8 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -218,6 +220,30 @@ public class IndexSearcher {
return (filter == null) ? query : new FilteredQuery(query, filter);
}
+ /**
+ * Count how many documents match the given query.
+ */
+ public int count(Query query) throws IOException {
+ final CollectorManager<TotalHitCountCollector, Integer> collectorManager = new CollectorManager<TotalHitCountCollector, Integer>() {
+
+ @Override
+ public TotalHitCountCollector newCollector() throws IOException {
+ return new TotalHitCountCollector();
+ }
+
+ @Override
+ public Integer reduce(Collection<TotalHitCountCollector> collectors) throws IOException {
+ int total = 0;
+ for (TotalHitCountCollector collector : collectors) {
+ total += collector.getTotalHits();
+ }
+ return total;
+ }
+
+ };
+ return search(query, collectorManager);
+ }
+
/** Finds the top <code>n</code>
* hits for <code>query</code> where all results are after a previous
* result (<code>after</code>).
@@ -229,7 +255,7 @@ public class IndexSearcher {
* @throws BooleanQuery.TooManyClauses If a query would exceed
* {@link BooleanQuery#getMaxClauseCount()} clauses.
*/
- public TopDocs searchAfter(ScoreDoc after, Query query, int numHits) throws IOException {
+ public TopDocs searchAfter(final ScoreDoc after, Query query, int numHits) throws IOException {
final int limit = Math.max(1, reader.maxDoc());
if (after != null && after.doc >= limit) {
throw new IllegalArgumentException("after.doc exceeds the number of documents in the reader: after.doc="
@@ -237,47 +263,30 @@ public class IndexSearcher {
}
numHits = Math.min(numHits, limit);
- if (executor == null) {
- final TopScoreDocCollector collector = TopScoreDocCollector.create(numHits, after);
- search(query, collector);
- return collector.topDocs();
- } else {
- final TopScoreDocCollector[] collectors = new TopScoreDocCollector[leafSlices.length];
- boolean needsScores = false;
- for (int i = 0; i < leafSlices.length; ++i) {
- collectors[i] = TopScoreDocCollector.create(numHits, after);
- needsScores |= collectors[i].needsScores();
- }
+ final int cappedNumHits = Math.min(numHits, limit);
- final Weight weight = createNormalizedWeight(query, needsScores);
- final List<Future<TopDocs>> topDocsFutures = new ArrayList<>(leafSlices.length);
- for (int i = 0; i < leafSlices.length; ++i) {
- final LeafReaderContext[] leaves = leafSlices[i].leaves;
- final TopScoreDocCollector collector = collectors[i];
- topDocsFutures.add(executor.submit(new Callable<TopDocs>() {
- @Override
- public TopDocs call() throws Exception {
- search(Arrays.asList(leaves), weight, collector);
- return collector.topDocs();
- }
- }));
+ final CollectorManager<TopScoreDocCollector, TopDocs> manager = new CollectorManager<TopScoreDocCollector, TopDocs>() {
+
+ @Override
+ public TopScoreDocCollector newCollector() throws IOException {
+ return TopScoreDocCollector.create(cappedNumHits, after);
}
- final TopDocs[] topDocs = new TopDocs[leafSlices.length];
- for (int i = 0; i < topDocs.length; ++i) {
- try {
- topDocs[i] = topDocsFutures.get(i).get();
- } catch (InterruptedException e) {
- throw new ThreadInterruptedException(e);
- } catch (ExecutionException e) {
- throw new RuntimeException(e);
+ @Override
+ public TopDocs reduce(Collection<TopScoreDocCollector> collectors) throws IOException {
+ final TopDocs[] topDocs = new TopDocs[collectors.size()];
+ int i = 0;
+ for (TopScoreDocCollector collector : collectors) {
+ topDocs[i++] = collector.topDocs();
}
+ return TopDocs.merge(cappedNumHits, topDocs);
}
- return TopDocs.merge(numHits, topDocs);
- }
+ };
+
+ return search(query, manager);
}
-
+
/** Finds the top <code>n</code>
* hits for <code>query</code>, applying <code>filter</code> if non-null,
* where all results are after a previous result (<code>after</code>).
@@ -446,7 +455,7 @@ public class IndexSearcher {
}
/** Finds the top <code>n</code>
- * hits for <code>query</code> where all results are after a previous
+ * hits for <code>query</code> where all results are after a previous
* result (<code>after</code>).
* <p>
* By passing the bottom result from a previous page as <code>after</code>,
@@ -461,7 +470,7 @@ public class IndexSearcher {
}
/** Finds the top <code>n</code>
- * hits for <code>query</code> where all results are after a previous
+ * hits for <code>query</code> where all results are after a previous
* result (<code>after</code>), allowing control over
* whether hit scores and max score should be computed.
* <p>
@@ -515,46 +524,79 @@ public class IndexSearcher {
return searchAfter((FieldDoc) after, query, numHits, sort, doDocScores, doMaxScore);
}
- private TopFieldDocs searchAfter(FieldDoc after, Query query, int numHits, Sort sort,
- boolean doDocScores, boolean doMaxScore) throws IOException {
+ private TopFieldDocs searchAfter(final FieldDoc after, Query query, int numHits, final Sort sort,
+ final boolean doDocScores, final boolean doMaxScore) throws IOException {
final int limit = Math.max(1, reader.maxDoc());
if (after != null && after.doc >= limit) {
throw new IllegalArgumentException("after.doc exceeds the number of documents in the reader: after.doc="
+ after.doc + " limit=" + limit);
}
- numHits = Math.min(numHits, limit);
+ final int cappedNumHits = Math.min(numHits, limit);
+
+ final CollectorManager<TopFieldCollector, TopFieldDocs> manager = new CollectorManager<TopFieldCollector, TopFieldDocs>() {
+
+ @Override
+ public TopFieldCollector newCollector() throws IOException {
+ final boolean fillFields = true;
+ return TopFieldCollector.create(sort, cappedNumHits, after, fillFields, doDocScores, doMaxScore);
+ }
+
+ @Override
+ public TopFieldDocs reduce(Collection<TopFieldCollector> collectors) throws IOException {
+ final TopFieldDocs[] topDocs = new TopFieldDocs[collectors.size()];
+ int i = 0;
+ for (TopFieldCollector collector : collectors) {
+ topDocs[i++] = collector.topDocs();
+ }
+ return TopDocs.merge(sort, cappedNumHits, topDocs);
+ }
+
+ };
+
+ return search(query, manager);
+ }
- final boolean fillFields = true;
+ /**
+ * Lower-level search API.
+ * Search all leaves using the given {@link CollectorManager}. In contrast
+ * to {@link #search(Query, Collector)}, this method will use the searcher's
+ * {@link ExecutorService} in order to parallelize execution of the collection
+ * on the configured {@link #leafSlices}.
+ * @see CollectorManager
+ * @lucene.experimental
+ */
+ public <C extends Collector, T> T search(Query query, CollectorManager<C, T> collectorManager) throws IOException {
if (executor == null) {
- final TopFieldCollector collector = TopFieldCollector.create(sort, numHits, after, fillFields, doDocScores, doMaxScore);
+ final C collector = collectorManager.newCollector();
search(query, collector);
- return collector.topDocs();
+ return collectorManager.reduce(Collections.singletonList(collector));
} else {
- final TopFieldCollector[] collectors = new TopFieldCollector[leafSlices.length];
+ final List<C> collectors = new ArrayList<>(leafSlices.length);
boolean needsScores = false;
for (int i = 0; i < leafSlices.length; ++i) {
- collectors[i] = TopFieldCollector.create(sort, numHits, after, fillFields, doDocScores, doMaxScore);
- needsScores |= collectors[i].needsScores();
+ final C collector = collectorManager.newCollector();
+ collectors.add(collector);
+ needsScores |= collector.needsScores();
}
final Weight weight = createNormalizedWeight(query, needsScores);
- final List<Future<TopFieldDocs>> topDocsFutures = new ArrayList<>(leafSlices.length);
+ final List<Future<C>> topDocsFutures = new ArrayList<>(leafSlices.length);
for (int i = 0; i < leafSlices.length; ++i) {
final LeafReaderContext[] leaves = leafSlices[i].leaves;
- final TopFieldCollector collector = collectors[i];
- topDocsFutures.add(executor.submit(new Callable<TopFieldDocs>() {
+ final C collector = collectors.get(i);
+ topDocsFutures.add(executor.submit(new Callable<C>() {
@Override
- public TopFieldDocs call() throws Exception {
+ public C call() throws Exception {
search(Arrays.asList(leaves), weight, collector);
- return collector.topDocs();
+ return collector;
}
}));
}
- final TopFieldDocs[] topDocs = new TopFieldDocs[leafSlices.length];
- for (int i = 0; i < topDocs.length; ++i) {
+ final List<C> collectedCollectors = new ArrayList<>();
+ for (Future<C> future : topDocsFutures) {
try {
- topDocs[i] = topDocsFutures.get(i).get();
+ collectedCollectors.add(future.get());
} catch (InterruptedException e) {
throw new ThreadInterruptedException(e);
} catch (ExecutionException e) {
@@ -562,7 +604,7 @@ public class IndexSearcher {
}
}
- return TopDocs.merge(sort, numHits, topDocs);
+ return collectorManager.reduce(collectors);
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java?rev=1662761&r1=1662760&r2=1662761&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java Fri Feb 27 17:37:31 2015
@@ -17,6 +17,7 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -24,7 +25,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -140,5 +143,26 @@ public class TestIndexSearcher extends L
IOUtils.close(r, dir);
}
}
-
+
+ public void testCount() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ if (random().nextBoolean()) {
+ doc.add(new StringField("foo", "bar", Store.NO));
+ }
+ w.addDocument(doc);
+ }
+ w.commit();
+ final IndexReader reader = w.getReader();
+ w.close();
+ final IndexSearcher searcher = newSearcher(reader);
+ final Query query = new TermQuery(new Term("foo", "bar"));
+ assertEquals(searcher.count(query), searcher.search(query, 1).totalHits);
+ reader.close();
+ dir.close();
+ }
+
}