You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2014/12/15 11:41:10 UTC
svn commit: r1645618 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/search/ lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/util/
Author: jpountz
Date: Mon Dec 15 10:41:10 2014
New Revision: 1645618
URL: http://svn.apache.org/r1645618
Log:
LUCENE-6106: Improve tracking of filter usage in LRUFilterCache.
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java?rev=1645618&r1=1645617&r2=1645618&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java Mon Dec 15 10:41:10 2014
@@ -38,7 +38,7 @@ public interface FilterCachingPolicy {
public static final FilterCachingPolicy ALWAYS_CACHE = new FilterCachingPolicy() {
@Override
- public void onCache(Filter filter) {}
+ public void onUse(Filter filter) {}
@Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
@@ -78,7 +78,7 @@ public interface FilterCachingPolicy {
}
@Override
- public void onCache(Filter filter) {}
+ public void onUse(Filter filter) {}
@Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
@@ -89,10 +89,10 @@ public interface FilterCachingPolicy {
};
- /** Callback that is called on every call to {@link FilterCache#doCache}.
+ /** Callback that is called every time that a cached filter is used.
* This is typically useful if the policy wants to track usage statistics
* in order to make decisions. */
- void onCache(Filter filter);
+ void onUse(Filter filter);
/** Whether the given {@link DocIdSet} should be cached on a given segment.
* This method will be called on each leaf context to know if the filter
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java?rev=1645618&r1=1645617&r2=1645618&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java Mon Dec 15 10:41:10 2014
@@ -345,6 +345,10 @@ public class LRUFilterCache implements F
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
+ if (context.ord == 0) {
+ policy.onUse(in);
+ }
+
DocIdSet set = get(in, context);
if (set == null) {
// do not apply acceptDocs yet, we want the cached filter to not take them into account
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java?rev=1645618&r1=1645617&r2=1645618&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingFilterCachingPolicy.java Mon Dec 15 10:41:10 2014
@@ -103,7 +103,7 @@ public final class UsageTrackingFilterCa
}
@Override
- public void onCache(Filter filter) {
+ public void onUse(Filter filter) {
// Using the filter hash codes might help keep memory usage a bit lower
// since some filters might have non-negligible memory usage?
recentlyUsedFilters.add(filter.hashCode());
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java?rev=1645618&r1=1645617&r2=1645618&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java Mon Dec 15 10:41:10 2014
@@ -22,6 +22,7 @@ import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -49,7 +50,7 @@ public class TestLRUFilterCache extends
private static final FilterCachingPolicy NEVER_CACHE = new FilterCachingPolicy() {
@Override
- public void onCache(Filter filter) {}
+ public void onUse(Filter filter) {}
@Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
@@ -398,6 +399,62 @@ public class TestLRUFilterCache extends
reader.close();
w.close();
+ dir.close();
+ }
+
+ public void testOnUse() throws IOException {
+ final LRUFilterCache filterCache = new LRUFilterCache(1 + random().nextInt(5), 1 + random().nextInt(1000));
+
+ Directory dir = newDirectory();
+ final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+
+ Document doc = new Document();
+ StringField f = new StringField("color", "", Store.NO);
+ doc.add(f);
+ final int numDocs = atLeast(10);
+ for (int i = 0; i < numDocs; ++i) {
+ f.setStringValue(RandomPicks.randomFrom(random(), Arrays.asList("red", "blue", "green", "yellow")));
+ w.addDocument(doc);
+ if (random().nextBoolean()) {
+ w.getReader().close();
+ }
+ }
+ final DirectoryReader reader = w.getReader();
+ final IndexSearcher searcher = new IndexSearcher(reader);
+
+ final Map<Filter, Integer> actualCounts = new HashMap<>();
+ final Map<Filter, Integer> expectedCounts = new HashMap<>();
+
+ final FilterCachingPolicy countingPolicy = new FilterCachingPolicy() {
+
+ @Override
+ public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
+ return random().nextBoolean();
+ }
+
+ @Override
+ public void onUse(Filter filter) {
+ expectedCounts.put(filter, 1 + (expectedCounts.containsKey(filter) ? expectedCounts.get(filter) : 0));
+ }
+ };
+
+ Filter[] filters = new Filter[10 + random().nextInt(10)];
+ Filter[] cachedFilters = new Filter[filters.length];
+ for (int i = 0; i < filters.length; ++i) {
+ filters[i] = new QueryWrapperFilter(new TermQuery(new Term("color", RandomPicks.randomFrom(random(), Arrays.asList("red", "blue", "green", "yellow")))));
+ cachedFilters[i] = filterCache.doCache(filters[i], countingPolicy);
+ }
+
+ for (int i = 0; i < 20; ++i) {
+ final int idx = random().nextInt(filters.length);
+ searcher.search(new ConstantScoreQuery(cachedFilters[idx]), 1);
+ actualCounts.put(filters[idx], 1 + (actualCounts.containsKey(filters[idx]) ? actualCounts.get(filters[idx]) : 0));
+ }
+
+ assertEquals(actualCounts, expectedCounts);
+
+ reader.close();
+ w.close();
dir.close();
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1645618&r1=1645617&r2=1645618&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Mon Dec 15 10:41:10 2014
@@ -474,7 +474,7 @@ public abstract class LuceneTestCase ext
public static final FilterCachingPolicy MAYBE_CACHE_POLICY = new FilterCachingPolicy() {
@Override
- public void onCache(Filter filter) {}
+ public void onUse(Filter filter) {}
@Override
public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {