You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2017/02/16 14:30:14 UTC
[4/4] lucene-solr:branch_6x: LUCENE-7677: Cache compound filters
earlier than regular filters.
LUCENE-7677: Cache compound filters earlier than regular filters.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1bfa057d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1bfa057d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1bfa057d
Branch: refs/heads/branch_6x
Commit: 1bfa057d5c9d89b116031baa7493ee422b4cbabb
Parents: 5682c33
Author: Adrien Grand <jp...@gmail.com>
Authored: Thu Feb 16 09:42:50 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Thu Feb 16 15:02:27 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
.../search/UsageTrackingQueryCachingPolicy.java | 12 ++-
.../TestUsageTrackingFilterCachingPolicy.java | 81 ++++++++++++++++++++
3 files changed, 96 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1bfa057d/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f63fb22..0e41e2f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -91,6 +91,10 @@ Improvements
since they are plenty fast. This also has the side-effect of leaving more
space in the history for costly filters. (Adrien Grand)
+* LUCENE-7677: UsageTrackingQueryCachingPolicy now caches compound queries a bit
+ earlier than regular queries in order to improve cache efficiency.
+ (Adrien Grand)
+
Optimizations
* LUCENE-7641: Optimized point range queries to compute documents that do not
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1bfa057d/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java b/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java
index 568931d..1bbed5c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java
@@ -122,7 +122,17 @@ public class UsageTrackingQueryCachingPolicy implements QueryCachingPolicy {
return 2;
} else {
// default: cache after the filter has been seen 5 times
- return 5;
+ int minFrequency = 5;
+ if (query instanceof BooleanQuery
+ || query instanceof DisjunctionMaxQuery) {
+ // Say you keep reusing a boolean query that looks like "A OR B" and
+ // never use the A and B queries out of that context. 5 times after it
+ // has been used, we would cache both A, B and A OR B, which is
+ // wasteful. So instead we cache compound queries a bit earlier so that
+ // we would only cache "A OR B" in that case.
+ minFrequency--;
+ }
+ return minFrequency;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1bfa057d/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
index eed3cb7..2d20904 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
@@ -18,8 +18,14 @@ package org.apache.lucene.search;
import java.io.IOException;
+import org.apache.lucene.document.Document;
import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
public class TestUsageTrackingFilterCachingPolicy extends LuceneTestCase {
@@ -48,4 +54,79 @@ public class TestUsageTrackingFilterCachingPolicy extends LuceneTestCase {
assertFalse(policy.shouldCache(q));
}
+ public void testBooleanQueries() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ w.addDocument(new Document());
+ IndexReader reader = w.getReader();
+ w.close();
+
+ IndexSearcher searcher = new IndexSearcher(reader);
+ UsageTrackingQueryCachingPolicy policy = new UsageTrackingQueryCachingPolicy();
+ LRUQueryCache cache = new LRUQueryCache(10, Long.MAX_VALUE, new LRUQueryCache.MinSegmentSizePredicate(1, 0f));
+ searcher.setQueryCache(cache);
+ searcher.setQueryCachingPolicy(policy);
+
+ DummyQuery q1 = new DummyQuery(1);
+ DummyQuery q2 = new DummyQuery(2);
+ BooleanQuery bq = new BooleanQuery.Builder()
+ .add(q1, Occur.SHOULD)
+ .add(q2, Occur.SHOULD)
+ .build();
+
+ for (int i = 0; i < 3; ++i) {
+ searcher.count(bq);
+ }
+ assertEquals(0, cache.getCacheSize()); // nothing cached yet, too early
+
+ searcher.count(bq);
+ assertEquals(1, cache.getCacheSize()); // the bq got cached, but not q1 and q2
+
+ for (int i = 0; i < 10; ++i) {
+ searcher.count(bq);
+ }
+ assertEquals(1, cache.getCacheSize()); // q1 and q2 still not cached since we do not pull scorers on them
+
+ searcher.count(q1);
+ assertEquals(2, cache.getCacheSize()); // q1 used on its own -> cached
+
+ reader.close();
+ dir.close();
+ }
+
+ private static class DummyQuery extends Query {
+
+ private final int id;
+
+ DummyQuery(int id) {
+ this.id = id;
+ }
+
+ @Override
+ public String toString(String field) {
+ return "dummy";
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return sameClassAs(obj) && ((DummyQuery) obj).id == id;
+ }
+
+ @Override
+ public int hashCode() {
+ return id;
+ }
+
+ @Override
+ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+ return new ConstantScoreWeight(DummyQuery.this) {
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(1));
+ }
+ };
+ }
+
+ }
+
}