You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/12/13 19:01:49 UTC
svn commit: r1213811 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/search/CachingWrapperFilter.java
test-framework/java/org/apache/lucene/search/CachingWrapperFilterHelper.java
test/org/apache/lucene/search/TestCachingWrapperFilter.java
Author: mikemccand
Date: Tue Dec 13 18:01:49 2011
New Revision: 1213811
URL: http://svn.apache.org/viewvc?rev=1213811&view=rev
Log:
LUCENE-3531: add boolean (default false) to CachingWrapperFilter to control whether it should recache on new deletes
Removed:
lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/search/CachingWrapperFilterHelper.java
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1213811&r1=1213810&r2=1213811&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java Tue Dec 13 18:01:49 2011
@@ -38,28 +38,27 @@ public class CachingWrapperFilter extend
Filter filter;
protected final FilterCache<DocIdSet> cache;
+ private final boolean recacheDeletes;
- static class FilterCache<T> {
+ private static class FilterCache<T> {
/**
* A transient Filter cache (package private because of test)
*/
- // NOTE: not final so that we can dynamically re-init
- // after de-serialize
- transient Map<Object,T> cache;
+ private final Map<Object,Map<Object,T>> cache = new WeakHashMap<Object,Map<Object,T>>();
- public synchronized T get(IndexReader reader, Object coreKey) throws IOException {
- T value;
-
- if (cache == null) {
- cache = new WeakHashMap<Object,T>();
+ public synchronized T get(IndexReader reader, Object coreKey, Object coreSubKey) throws IOException {
+ Map<Object,T> innerCache = cache.get(coreKey);
+ if (innerCache == null) {
+ innerCache = new WeakHashMap<Object,T>();
+ cache.put(coreKey, innerCache);
}
- return cache.get(coreKey);
+ return innerCache.get(coreSubKey);
}
- public synchronized void put(Object coreKey, T value) {
- cache.put(coreKey, value);
+ public synchronized void put(Object coreKey, Object coreSubKey, T value) {
+ cache.get(coreKey).put(coreSubKey, value);
}
}
@@ -67,7 +66,19 @@ public class CachingWrapperFilter extend
* @param filter Filter to cache results of
*/
public CachingWrapperFilter(Filter filter) {
+ this(filter, false);
+ }
+
+ /** Wraps another filter's result and caches it. If
+ * recacheDeletes is true, then new deletes (for example
+ * after {@link IndexReader#openIfChanged}) will be AND'd
+ * and cached again.
+ *
+ * @param filter Filter to cache results of
+ */
+ public CachingWrapperFilter(Filter filter, boolean recacheDeletes) {
this.filter = filter;
+ this.recacheDeletes = recacheDeletes;
cache = new FilterCache<DocIdSet>();
}
@@ -106,33 +117,48 @@ public class CachingWrapperFilter extend
final IndexReader reader = context.reader;
final Object coreKey = reader.getCoreCacheKey();
- DocIdSet docIdSet = cache.get(reader, coreKey);
+ // Only cache if incoming acceptDocs is == live docs;
+ // if Lucene passes in more interesting acceptDocs in
+ // the future we don't want to over-cache:
+ final boolean doCacheSubAcceptDocs = recacheDeletes && acceptDocs == reader.getLiveDocs();
+
+ final Bits subAcceptDocs;
+ if (doCacheSubAcceptDocs) {
+ subAcceptDocs = acceptDocs;
+ } else {
+ subAcceptDocs = null;
+ }
+
+ DocIdSet docIdSet = cache.get(reader, coreKey, subAcceptDocs);
if (docIdSet != null) {
hitCount++;
} else {
missCount++;
- // cache miss: we use no acceptDocs here
- // (this saves time on building DocIdSet, the acceptDocs will be applied on the cached set)
- docIdSet = docIdSetToCache(filter.getDocIdSet(context, null/**!!!*/), reader);
- cache.put(coreKey, docIdSet);
+ docIdSet = docIdSetToCache(filter.getDocIdSet(context, subAcceptDocs), reader);
+ cache.put(coreKey, subAcceptDocs, docIdSet);
+ }
+
+ if (doCacheSubAcceptDocs) {
+ return docIdSet;
+ } else {
+ return BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
}
-
- return BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
}
@Override
public String toString() {
- return "CachingWrapperFilter("+filter+")";
+ return "CachingWrapperFilter("+filter+",recacheDeletes=" + recacheDeletes + ")";
}
@Override
public boolean equals(Object o) {
if (!(o instanceof CachingWrapperFilter)) return false;
- return this.filter.equals(((CachingWrapperFilter)o).filter);
+ final CachingWrapperFilter other = (CachingWrapperFilter) o;
+ return this.filter.equals(other.filter) && this.recacheDeletes == other.recacheDeletes;
}
@Override
public int hashCode() {
- return filter.hashCode() ^ 0x1117BF25;
+ return (filter.hashCode() ^ 0x1117BF25) + (recacheDeletes ? 0 : 1);
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1213811&r1=1213810&r2=1213811&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Tue Dec 13 18:01:49 2011
@@ -30,8 +30,9 @@ import org.apache.lucene.index.SlowMulti
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TestCachingWrapperFilter extends LuceneTestCase {
@@ -164,6 +165,7 @@ public class TestCachingWrapperFilter ex
// asserts below requires no unexpected merges:
setMergePolicy(newLogMergePolicy(10))
);
+ _TestUtil.keepFullyDeletedSegments(writer.w);
// NOTE: cannot use writer.getReader because RIW (on
// flipping a coin) may give us a newly opened reader,
@@ -173,7 +175,7 @@ public class TestCachingWrapperFilter ex
// same reason we don't wrap?
IndexSearcher searcher = newSearcher(reader, false);
- // add a doc, refresh the reader, and check that its there
+ // add a doc, refresh the reader, and check that it's there
Document doc = new Document();
doc.add(newField("id", "1", StringField.TYPE_STORED));
writer.addDocument(doc);
@@ -186,25 +188,78 @@ public class TestCachingWrapperFilter ex
final Filter startFilter = new QueryWrapperFilter(new TermQuery(new Term("id", "1")));
- CachingWrapperFilter filter = new CachingWrapperFilter(startFilter);
+ // force cache to regenerate after deletions:
+ CachingWrapperFilter filter = new CachingWrapperFilter(startFilter, true);
docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+
assertEquals("[query + filter] Should find a hit...", 1, docs.totalHits);
- int missCount = filter.missCount;
- assertTrue(missCount > 0);
+
Query constantScore = new ConstantScoreQuery(filter);
docs = searcher.search(constantScore, 1);
assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+
+ // make sure we get a cache hit when we reopen reader
+ // that had no change to deletions
+
+ // fake delete (deletes nothing):
+ writer.deleteDocuments(new Term("foo", "bar"));
+
+ IndexReader oldReader = reader;
+ reader = refreshReader(reader);
+ assertTrue(reader == oldReader);
+ int missCount = filter.missCount;
+ docs = searcher.search(constantScore, 1);
+ assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+
+ // cache hit:
assertEquals(missCount, filter.missCount);
+ // now delete the doc, refresh the reader, and see that it's not there
+ writer.deleteDocuments(new Term("id", "1"));
+
// NOTE: important to hold ref here so GC doesn't clear
// the cache entry! Else the assert below may sometimes
// fail:
- IndexReader oldReader = reader;
+ oldReader = reader;
+ reader = refreshReader(reader);
+
+ searcher = newSearcher(reader, false);
+
+ missCount = filter.missCount;
+ docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+ assertEquals("[query + filter] Should *not* find a hit...", 0, docs.totalHits);
+ // cache miss, because we asked CWF to recache when
+ // deletes changed:
+ assertEquals(missCount+1, filter.missCount);
+ docs = searcher.search(constantScore, 1);
+ assertEquals("[just filter] Should *not* find a hit...", 0, docs.totalHits);
+
+ // apply deletes dynamically:
+ filter = new CachingWrapperFilter(startFilter);
writer.addDocument(doc);
reader = refreshReader(reader);
searcher = newSearcher(reader, false);
+
+ docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+ assertEquals("[query + filter] Should find a hit...", 1, docs.totalHits);
+ missCount = filter.missCount;
+ assertTrue(missCount > 0);
+ constantScore = new ConstantScoreQuery(filter);
+ docs = searcher.search(constantScore, 1);
+ assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+ assertEquals(missCount, filter.missCount);
+
+ writer.addDocument(doc);
+
+ // NOTE: important to hold ref here so GC doesn't clear
+ // the cache entry! Else the assert below may sometimes
+ // fail:
+ oldReader = reader;
+
+ reader = refreshReader(reader);
+ searcher = newSearcher(reader, false);
docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
assertEquals("[query + filter] Should find 2 hits...", 2, docs.totalHits);
@@ -216,11 +271,6 @@ public class TestCachingWrapperFilter ex
assertEquals("[just filter] Should find a hit...", 2, docs.totalHits);
assertEquals(missCount, filter.missCount);
- // NOTE: important to hold ref here so GC doesn't clear
- // the cache entry! Else the assert below may sometimes
- // fail:
- IndexReader oldReader2 = reader;
-
// now delete the doc, refresh the reader, and see that it's not there
writer.deleteDocuments(new Term("id", "1"));
@@ -229,10 +279,12 @@ public class TestCachingWrapperFilter ex
docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
assertEquals("[query + filter] Should *not* find a hit...", 0, docs.totalHits);
+ // CWF reused the same entry (it dynamically applied the deletes):
assertEquals(missCount, filter.missCount);
docs = searcher.search(constantScore, 1);
assertEquals("[just filter] Should *not* find a hit...", 0, docs.totalHits);
+ // CWF reused the same entry (it dynamically applied the deletes):
assertEquals(missCount, filter.missCount);
// NOTE: silliness to make sure JRE does not eliminate
@@ -240,7 +292,6 @@ public class TestCachingWrapperFilter ex
// CachingWrapperFilter's WeakHashMap from dropping the
// entry:
assertTrue(oldReader != null);
- assertTrue(oldReader2 != null);
reader.close();
writer.close();