svn commit: r1676607 - in /lucene/dev/trunk/lucene: CHANGES.txt core/src/java/org/apache/lucene/search/LRUQueryCache.java core/src/test/org/apache/lucene/search/TestLRUQueryCache.java

[jp...@apache.org]

Author: jpountz
Date: Tue Apr 28 20:09:41 2015
New Revision: 1676607

URL: http://svn.apache.org/r1676607
Log:
LUCENE-6456: Do not cache queries that are too large for the query cache.

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1676607&r1=1676606&r2=1676607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Apr 28 20:09:41 2015
@@ -101,6 +101,9 @@ Optimizations
 * LUCENE-6392: Highligher- reduce memory of tokens in
   TokenStreamFromTermVector, and add maxStartOffset limit. (David Smiley)
 
+* LUCENE-6456: Queries that generate doc id sets that are too large for the
+  query cache are not cached instead of evicting everything. (Adrien Grand)
+
 Bug Fixes
 
 * LUCENE-6378: Fix all RuntimeExceptions to throw the underlying root cause.

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java?rev=1676607&r1=1676606&r2=1676607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java Tue Apr 28 20:09:41 2015
@@ -30,6 +30,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.LeafReader.CoreClosedListener;
 import org.apache.lucene.index.LeafReaderContext;
@@ -552,6 +553,18 @@ public class LRUQueryCache implements Qu
       in.extractTerms(terms);
     }
 
+    private boolean cacheEntryHasReasonableWorstCaseSize(int maxDoc) {
+      // The worst-case (dense) is a bit set which needs one bit per document
+      final long worstCaseRamUsage = maxDoc / 8;
+      final long totalRamAvailable = maxRamBytesUsed;
+      // Imagine the worst-case that a cache entry is large than the size of
+      // the cache: not only will this entry be trashed immediately but it
+      // will also evict all current entries from the cache. For this reason
+      // we only cache on an IndexReader if we have available room for
+      // 5 different filters on this reader to avoid excessive trashing
+      return worstCaseRamUsage * 5 < totalRamAvailable;
+    }
+
     @Override
     protected Scorer scorer(LeafReaderContext context, Bits acceptDocs, float score) throws IOException {
       if (context.ord == 0) {
@@ -559,7 +572,8 @@ public class LRUQueryCache implements Qu
       }
       DocIdSet docIdSet = get(in.getQuery(), context);
       if (docIdSet == null) {
-        if (policy.shouldCache(in.getQuery(), context)) {
+        if (cacheEntryHasReasonableWorstCaseSize(ReaderUtil.getTopLevelContext(context).reader().maxDoc())
+            && policy.shouldCache(in.getQuery(), context)) {
           final Scorer scorer = in.scorer(context, null);
           if (scorer == null) {
             docIdSet = DocIdSet.EMPTY;

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java?rev=1676607&r1=1676606&r2=1676607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java Tue Apr 28 20:09:41 2015
@@ -984,4 +984,27 @@ public class TestLRUQueryCache extends L
     
     IOUtils.close(w, reader, dir);
   }
+
+  public void testRefuseToCacheTooLargeEntries() throws IOException {
+    Directory dir = newDirectory();
+    final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+    for (int i = 0; i < 100; ++i) {
+      w.addDocument(new Document());
+    }
+    IndexReader reader = w.getReader();
+
+    // size of 1 byte
+    final LRUQueryCache queryCache = new LRUQueryCache(1, 1);
+    final IndexSearcher searcher = newSearcher(reader);
+    searcher.setQueryCache(queryCache);
+    searcher.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
+
+    searcher.count(new MatchAllDocsQuery());
+    assertEquals(0, queryCache.getCacheCount());
+    assertEquals(0, queryCache.getEvictionCount());
+
+    reader.close();
+    w.close();
+    dir.close();
+  }
 }