You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2015/09/02 14:48:04 UTC
svn commit: r1700793 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/search/
Author: jpountz
Date: Wed Sep 2 12:47:56 2015
New Revision: 1700793
URL: http://svn.apache.org/r1700793
Log:
LUCENE-6754: Optimized IndexSearcher.count for simple queries.
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1700793&r1=1700792&r2=1700793&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Wed Sep 2 12:47:56 2015
@@ -36,6 +36,9 @@ Optimizations
* LUCENE-6746: DisjunctionMaxQuery, BoostingQuery and BoostedQuery now create
sub weights through IndexSearcher so that they can be cached. (Adrien Grand)
+* LUCENE-6754: Optimized IndexSearcher.count for the cases when it can use
+ index statistics instead of collecting all matches. (Adrien Grand)
+
Bug Fixes
* LUCENE-6730: Hyper-parameter c is ignored in term frequency NormalizationH1.
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1700793&r1=1700792&r2=1700793&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java Wed Sep 2 12:47:56 2015
@@ -348,6 +348,29 @@ public class IndexSearcher {
* Count how many documents match the given query.
*/
public int count(Query query) throws IOException {
+ query = rewrite(query);
+ while (true) {
+ // remove wrappers that don't matter for counts
+ if (query instanceof ConstantScoreQuery) {
+ query = ((ConstantScoreQuery) query).getQuery();
+ } else {
+ break;
+ }
+ }
+
+ // some counts can be computed in constant time
+ if (query instanceof MatchAllDocsQuery) {
+ return reader.numDocs();
+ } else if (query instanceof TermQuery && reader.hasDeletions() == false) {
+ Term term = ((TermQuery) query).getTerm();
+ int count = 0;
+ for (LeafReaderContext leaf : reader.leaves()) {
+ count += leaf.reader().docFreq(term);
+ }
+ return count;
+ }
+
+ // general case: create a collecor and count matches
final CollectorManager<TotalHitCountCollector, Integer> collectorManager = new CollectorManager<TotalHitCountCollector, Integer>() {
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java?rev=1700793&r1=1700792&r2=1700793&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestIndexSearcher.java Wed Sep 2 12:47:56 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.Arrays;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -31,6 +32,7 @@ import org.apache.lucene.document.String
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -42,7 +44,7 @@ import org.junit.Test;
public class TestIndexSearcher extends LuceneTestCase {
Directory dir;
IndexReader reader;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
@@ -58,20 +60,20 @@ public class TestIndexSearcher extends L
reader = iw.getReader();
iw.close();
}
-
+
@Override
public void tearDown() throws Exception {
super.tearDown();
reader.close();
dir.close();
}
-
+
// should not throw exception
public void testHugeN() throws Exception {
ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("TestIndexSearcher"));
-
+
IndexSearcher searchers[] = new IndexSearcher[] {
new IndexSearcher(reader),
new IndexSearcher(reader, service)
@@ -92,7 +94,7 @@ public class TestIndexSearcher extends L
null,
new FieldDoc(0, 0f, new Object[] { new BytesRef("boo!") })
};
-
+
for (IndexSearcher searcher : searchers) {
for (ScoreDoc after : afters) {
for (Query query : queries) {
@@ -120,10 +122,10 @@ public class TestIndexSearcher extends L
}
}
}
-
+
TestUtil.shutdownExecutorService(service);
}
-
+
@Test
public void testSearchAfterPassedMaxDoc() throws Exception {
// LUCENE-5128: ensure we get a meaningful message if searchAfter exceeds maxDoc
@@ -132,7 +134,7 @@ public class TestIndexSearcher extends L
w.addDocument(new Document());
IndexReader r = w.getReader();
w.close();
-
+
IndexSearcher s = new IndexSearcher(r);
try {
s.searchAfter(new ScoreDoc(r.maxDoc(), 0.54f), new MatchAllDocsQuery(), 10);
@@ -153,15 +155,36 @@ public class TestIndexSearcher extends L
if (random().nextBoolean()) {
doc.add(new StringField("foo", "bar", Store.NO));
}
+ if (random().nextBoolean()) {
+ doc.add(new StringField("foo", "baz", Store.NO));
+ }
+ if (rarely()) {
+ doc.add(new StringField("delete", "yes", Store.NO));
+ }
w.addDocument(doc);
}
- w.commit();
- final IndexReader reader = w.getReader();
+ for (boolean delete : new boolean[] {false, true}) {
+ if (delete) {
+ w.deleteDocuments(new Term("delete", "yes"));
+ }
+ final IndexReader reader = w.getReader();
+ final IndexSearcher searcher = newSearcher(reader);
+ // Test multiple queries, some of them are optimized by IndexSearcher.count()
+ for (Query query : Arrays.asList(
+ new MatchAllDocsQuery(),
+ new MatchNoDocsQuery(),
+ new TermQuery(new Term("foo", "bar")),
+ new ConstantScoreQuery(new TermQuery(new Term("foo", "baz"))),
+ new BooleanQuery.Builder()
+ .add(new TermQuery(new Term("foo", "bar")), Occur.SHOULD)
+ .add(new TermQuery(new Term("foo", "baz")), Occur.SHOULD)
+ .build()
+ )) {
+ assertEquals(searcher.count(query), searcher.search(query, 1).totalHits);
+ }
+ reader.close();
+ }
w.close();
- final IndexSearcher searcher = newSearcher(reader);
- final Query query = new TermQuery(new Term("foo", "bar"));
- assertEquals(searcher.count(query), searcher.search(query, 1).totalHits);
- reader.close();
dir.close();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java?rev=1700793&r1=1700792&r2=1700793&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java Wed Sep 2 12:47:56 2015
@@ -991,7 +991,7 @@ public class TestLRUQueryCache extends L
try {
// trigger an eviction
- searcher.count(new MatchAllDocsQuery());
+ searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollector());
fail();
} catch (ConcurrentModificationException e) {
// expected