You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2015/02/27 19:55:27 UTC
svn commit: r1662791 [1/2] - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/index/ lucene/core/src/test/org/...
Author: jpountz
Date: Fri Feb 27 18:55:26 2015
New Revision: 1662791
URL: http://svn.apache.org/r1662791
Log:
LUCENE-6303: CachingWrapperFilter -> CachingWrapperQuery, FilterCache -> QueryCache and added caching to IndexSearcher.
Added:
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java
- copied, changed from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
- copied, changed from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/QueryCache.java
- copied, changed from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/QueryCache.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/QueryCachingPolicy.java
- copied unchanged from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/QueryCachingPolicy.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java
- copied, changed from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java
- copied unchanged from r1662774, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
- copied, changed from r1662774, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCache.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilterCachingPolicy.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
lucene/dev/branches/branch_5x/lucene/facet/ (props changed)
lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java
lucene/dev/branches/branch_5x/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
lucene/dev/branches/branch_5x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java
lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
lucene/dev/branches/branch_5x/lucene/join/ (props changed)
lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java
lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java
lucene/dev/branches/branch_5x/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java
lucene/dev/branches/branch_5x/lucene/misc/ (props changed)
lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestBlockJoinSorter.java
lucene/dev/branches/branch_5x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_5x/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/CachedFilterBuilder.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Fri Feb 27 18:55:26 2015
@@ -19,6 +19,9 @@ New Features
* LUCENE-6294: Added oal.search.CollectorManager to allow for parallelization
of the document collection process on IndexSearcher. (Adrien Grand)
+* LUCENE-6303: Added filter caching baked into IndexSearcher and enabled by
+ default. (Adrien Grand)
+
Bug Fixes
* LUCENE-6249: StandardQueryParser doesn't support pure negative clauses.
@@ -147,6 +150,9 @@ API Changes
removed. Use PrefixQuery, TermRangeQuery and NumericRangeQuery instead.
(Adrien Grand)
+* LUCENE-6303: Replaced FilterCache with QueryCache and CachingWrapperFilter
+ with CachingWrapperQuery. (Adrien Grand)
+
Other
* LUCENE-6248: Remove unused odd constants from StandardSyntaxParser.jj
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java Fri Feb 27 18:55:26 2015
@@ -20,7 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.CachingWrapperQuery;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -38,7 +38,7 @@ import org.apache.lucene.util.BytesRef;
* <p><b>NOTE</b>: If this {@link FilterLeafReader} does not change the
* content the contained reader, you could consider overriding
* {@link #getCoreCacheKey()} so that
- * {@link CachingWrapperFilter} shares the same entries for this atomic reader
+ * {@link CachingWrapperQuery} shares the same entries for this atomic reader
* and the wrapped one. {@link #getCombinedCoreAndDeletesKey()} could be
* overridden as well if the {@link #getLiveDocs() live docs} are not changed
* either.
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Fri Feb 27 18:55:26 2015
@@ -25,6 +25,7 @@ import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.ToStringUtils;
@@ -158,9 +159,27 @@ public class BooleanQuery extends Query
@Override
public final Iterator<BooleanClause> iterator() { return clauses().iterator(); }
+ private static BooleanQuery downgradeMustClauseToFilter(BooleanQuery bq) {
+ BooleanQuery clone = bq.clone();
+ clone.clauses.clear();
+ for (BooleanClause clause : bq.clauses()) {
+ if (clause.getOccur() == Occur.MUST) {
+ clone.add(clause.getQuery(), Occur.FILTER);
+ } else {
+ clone.add(clause);
+ }
+ }
+ return clone;
+ }
+
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
- return new BooleanWeight(this, searcher, needsScores, disableCoord);
+ BooleanQuery query = this;
+ if (needsScores == false) {
+ // we rewrite MUST clauses to FILTER for caching
+ query = downgradeMustClauseToFilter(query);
+ }
+ return new BooleanWeight(query, searcher, needsScores, disableCoord);
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java Fri Feb 27 18:55:26 2015
@@ -52,7 +52,7 @@ public class BooleanWeight extends Weigh
weights = new ArrayList<>(query.clauses().size());
for (int i = 0 ; i < query.clauses().size(); i++) {
BooleanClause c = query.clauses().get(i);
- Weight w = c.getQuery().createWeight(searcher, needsScores && c.isScoring());
+ Weight w = searcher.createWeight(c.getQuery(), needsScores && c.isScoring());
weights.add(w);
if (c.isScoring()) {
maxCoord++;
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java Fri Feb 27 18:55:26 2015
@@ -38,7 +38,9 @@ import org.apache.lucene.util.RoaringDoc
* Wraps another {@link Filter}'s result and caches it. The purpose is to allow
* filters to simply filter, and then wrap with this class
* to add caching.
+ * @deprecated Use {@link CachingWrapperQuery} and {@link BooleanClause.Occur#FILTER} clauses instead
*/
+@Deprecated
public class CachingWrapperFilter extends Filter implements Accountable {
private final Filter filter;
private final FilterCachingPolicy policy;
Copied: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java (from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java?p2=lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java&r1=1662774&r2=1662791&rev=1662791&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/CachingWrapperQuery.java Fri Feb 27 18:55:26 2015
@@ -116,7 +116,7 @@ public class CachingWrapperQuery extends
policy.onUse(weight.getQuery());
return new ConstantScoreWeight(weight.getQuery()) {
@Override
- Scorer scorer(LeafReaderContext context, Bits acceptDocs, float score) throws IOException {
+ Scorer scorer(LeafReaderContext context, final Bits acceptDocs, float score) throws IOException {
final LeafReader reader = context.reader();
final Object key = reader.getCoreCacheKey();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java Fri Feb 27 18:55:26 2015
@@ -256,7 +256,7 @@ public class ConstantScoreQuery extends
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
- final Weight innerWeight = query.createWeight(searcher, false);
+ final Weight innerWeight = searcher.createWeight(query, false);
if (needsScores) {
return new ConstantScoreQuery.ConstantWeight(innerWeight);
} else {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCache.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCache.java Fri Feb 27 18:55:26 2015
@@ -22,7 +22,9 @@ package org.apache.lucene.search;
*
* @see LRUFilterCache
* @lucene.experimental
+ * @deprecated Use {@link QueryCache} instead
*/
+@Deprecated
public interface FilterCache {
/**
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilterCachingPolicy.java Fri Feb 27 18:55:26 2015
@@ -32,8 +32,9 @@ import org.apache.lucene.index.TieredMer
* @see UsageTrackingFilterCachingPolicy
* @see LRUFilterCache
* @lucene.experimental
+ * @deprecated Use {@link QueryCachingPolicy} instead
*/
-// TODO: add APIs for integration with IndexWriter.IndexReaderWarmer
+@Deprecated
public interface FilterCachingPolicy {
/** A simple policy that caches all the provided filters on all segments. */
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java Fri Feb 27 18:55:26 2015
@@ -37,7 +37,7 @@ import org.apache.lucene.util.ToStringUt
* query is used in a search - use a CachingWrapperFilter to avoid
* regenerating the bits every time.
* @since 1.4
- * @see CachingWrapperFilter
+ * @see CachingWrapperQuery
*/
public class FilteredQuery extends Query {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java Fri Feb 27 18:55:26 2015
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -73,6 +74,11 @@ import org.apache.lucene.util.ThreadInte
* use your own (non-Lucene) objects instead.</p>
*/
public class IndexSearcher {
+
+ // 32MB and at most 10,000 queries
+ private static final QueryCache DEFAULT_QUERY_CACHE = new LRUQueryCache(10000, 1 << 25);
+ private static final QueryCachingPolicy DEFAULT_CACHING_POLICY = new UsageTrackingQueryCachingPolicy();
+
final IndexReader reader; // package private for testing!
// NOTE: these members might change in incompatible ways
@@ -87,7 +93,10 @@ public class IndexSearcher {
// the default Similarity
private static final Similarity defaultSimilarity = new DefaultSimilarity();
-
+
+ private QueryCache queryCache = DEFAULT_QUERY_CACHE;
+ private QueryCachingPolicy queryCachingPolicy = DEFAULT_CACHING_POLICY;
+
/**
* Expert: returns a default Similarity instance.
* In general, this method is only called to initialize searchers and writers.
@@ -157,7 +166,28 @@ public class IndexSearcher {
public IndexSearcher(IndexReaderContext context) {
this(context, null);
}
-
+
+ /**
+ * Set the {@link QueryCache} to use when scores are not needed.
+ * A value of {@code null} indicates that query matches should never be
+ * cached. This method should be called <b>before</b> starting using this
+ * {@link IndexSearcher}.
+ * @see QueryCache
+ */
+ public void setQueryCache(QueryCache queryCache) {
+ this.queryCache = queryCache;
+ }
+
+ /**
+ * Set the {@link QueryCachingPolicy} to use for query caching.
+ * This method should be called <b>before</b> starting using this
+ * {@link IndexSearcher}.
+ * @see QueryCachingPolicy
+ */
+ public void setQueryCachingPolicy(QueryCachingPolicy queryCachingPolicy) {
+ this.queryCachingPolicy = Objects.requireNonNull(queryCachingPolicy);
+ }
+
/**
* Expert: Creates an array of leaf slices each holding a subset of the given leaves.
* Each {@link LeafSlice} is executed in a single thread. By default there
@@ -708,7 +738,7 @@ public class IndexSearcher {
*/
public Weight createNormalizedWeight(Query query, boolean needsScores) throws IOException {
query = rewrite(query);
- Weight weight = query.createWeight(this, needsScores);
+ Weight weight = createWeight(query, needsScores);
float v = weight.getValueForNormalization();
float norm = getSimilarity().queryNorm(v);
if (Float.isInfinite(norm) || Float.isNaN(norm)) {
@@ -717,7 +747,21 @@ public class IndexSearcher {
weight.normalize(norm, 1.0f);
return weight;
}
-
+
+ /**
+ * Creates a {@link Weight} for the given query, potentially adding caching
+ * if possible and configured.
+ * @lucene.experimental
+ */
+ public Weight createWeight(Query query, boolean needsScores) throws IOException {
+ final QueryCache queryCache = this.queryCache;
+ Weight weight = query.createWeight(this, needsScores);
+ if (needsScores == false && queryCache != null) {
+ weight = queryCache.doCache(weight, queryCachingPolicy);
+ }
+ return weight;
+ }
+
/**
* Returns this searchers the top-level {@link IndexReaderContext}.
* @see IndexReader#getContext()
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUFilterCache.java Fri Feb 27 18:55:26 2015
@@ -82,7 +82,9 @@ import org.apache.lucene.util.RoaringDoc
*
* @see FilterCachingPolicy
* @lucene.experimental
+ * @deprecated Use {@link LRUQueryCache} instead
*/
+@Deprecated
public class LRUFilterCache implements FilterCache, Accountable {
// memory usage of a simple query-wrapper filter around a term query
Copied: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java (from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java?p2=lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java&r1=1662774&r2=1662791&rev=1662791&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java Fri Feb 27 18:55:26 2015
@@ -85,6 +85,17 @@ import org.apache.lucene.util.RoaringDoc
*/
public class LRUQueryCache implements QueryCache, Accountable {
+ private static Query cacheKey(Query query) {
+ if (query.getBoost() == 1f) {
+ return query;
+ } else {
+ Query key = query.clone();
+ key.setBoost(1f);
+ assert key == cacheKey(key);
+ return key;
+ }
+ }
+
// memory usage of a simple term query
static final long QUERY_DEFAULT_RAM_BYTES_USED = 192;
@@ -213,7 +224,7 @@ public class LRUQueryCache implements Qu
}
synchronized DocIdSet get(Query key, LeafReaderContext context) {
- key = QueryCache.cacheKey(key);
+ key = cacheKey(key);
final Object readerKey = context.reader().getCoreCacheKey();
final LeafCache leafCache = cache.get(readerKey);
if (leafCache == null) {
@@ -240,7 +251,7 @@ public class LRUQueryCache implements Qu
// we don't want to have user-provided queries as keys in our cache since queries are mutable
query = query.clone();
query.setBoost(1f);
- assert query == QueryCache.cacheKey(query);
+ assert query == cacheKey(query);
Query singleton = uniqueQueries.putIfAbsent(query, query);
if (singleton == null) {
onQueryCache(singleton, LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY + ramBytesUsed(query));
@@ -293,7 +304,7 @@ public class LRUQueryCache implements Qu
* Remove all cache entries for the given query.
*/
public synchronized void clearQuery(Query query) {
- final Query singleton = uniqueQueries.remove(QueryCache.cacheKey(query));
+ final Query singleton = uniqueQueries.remove(cacheKey(query));
if (singleton != null) {
onEviction(singleton);
}
@@ -322,7 +333,7 @@ public class LRUQueryCache implements Qu
+ ", maxSize=" + maxSize + ", ramBytesUsed=" + ramBytesUsed() + ", maxRamBytesUsed=" + maxRamBytesUsed);
}
for (LeafCache leafCache : cache.values()) {
- Set<Query> keys = Collections.newSetFromMap(new IdentityHashMap<>());
+ Set<Query> keys = Collections.newSetFromMap(new IdentityHashMap<Query, Boolean>());
keys.addAll(leafCache.cache.keySet());
keys.removeAll(mostRecentlyUsedQueries);
if (!keys.isEmpty()) {
@@ -497,12 +508,12 @@ public class LRUQueryCache implements Qu
}
DocIdSet get(Query query) {
- assert query == QueryCache.cacheKey(query);
+ assert query == cacheKey(query);
return cache.get(query);
}
void putIfAbsent(Query query, DocIdSet set) {
- assert query == QueryCache.cacheKey(query);
+ assert query == cacheKey(query);
if (cache.putIfAbsent(query, set) == null) {
// the set was actually put
onDocIdSetCache(HASHTABLE_RAM_BYTES_PER_ENTRY + set.ramBytesUsed());
@@ -510,7 +521,7 @@ public class LRUQueryCache implements Qu
}
void remove(Query query) {
- assert query == QueryCache.cacheKey(query);
+ assert query == cacheKey(query);
DocIdSet removed = cache.remove(query);
if (removed != null) {
onDocIdSetEviction(HASHTABLE_RAM_BYTES_PER_ENTRY + removed.ramBytesUsed());
@@ -522,6 +533,10 @@ public class LRUQueryCache implements Qu
return ramBytesUsed;
}
+ @Override
+ public Collection<Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
}
private class CachingWrapperWeight extends ConstantScoreWeight {
@@ -536,7 +551,7 @@ public class LRUQueryCache implements Qu
}
@Override
- Scorer scorer(LeafReaderContext context, Bits acceptDocs, float score) throws IOException {
+ Scorer scorer(LeafReaderContext context, final Bits acceptDocs, float score) throws IOException {
if (context.ord == 0) {
policy.onUse(getQuery());
}
Copied: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/QueryCache.java (from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/QueryCache.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/QueryCache.java?p2=lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/QueryCache.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/QueryCache.java&r1=1662774&r2=1662791&rev=1662791&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/QueryCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/QueryCache.java Fri Feb 27 18:55:26 2015
@@ -26,22 +26,6 @@ package org.apache.lucene.search;
public interface QueryCache {
/**
- * Return a key for the given query that only takes matching documents into
- * account. Boosts will be ignored.
- * @lucene.internal
- */
- public static Query cacheKey(Query query) {
- if (query.getBoost() == 1f) {
- return query;
- } else {
- Query key = query.clone();
- key.setBoost(1f);
- assert key == cacheKey(key);
- return key;
- }
- }
-
- /**
* Return a wrapper around the provided <code>weight</code> that will cache
* matching docs per-segment accordingly to the given <code>policy</code>.
* NOTE: The returned weight will only be equivalent if scores are not needed.
Copied: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java (from r1662774, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java?p2=lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java&r1=1662774&r2=1662791&rev=1662791&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/UsageTrackingQueryCachingPolicy.java Fri Feb 27 18:55:26 2015
@@ -35,6 +35,17 @@ import org.apache.lucene.util.FrequencyT
*/
public final class UsageTrackingQueryCachingPolicy implements QueryCachingPolicy {
+ private static Query cacheKey(Query query) {
+ if (query.getBoost() == 1f) {
+ return query;
+ } else {
+ Query key = query.clone();
+ key.setBoost(1f);
+ assert key == cacheKey(key);
+ return key;
+ }
+ }
+
// the hash code that we use as a sentinel in the ring buffer.
private static final int SENTINEL = Integer.MIN_VALUE;
@@ -98,12 +109,12 @@ public final class UsageTrackingQueryCac
public void onUse(Query query) {
// we only track hash codes, which
synchronized (this) {
- recentlyUsedFilters.add(QueryCache.cacheKey(query).hashCode());
+ recentlyUsedFilters.add(cacheKey(query).hashCode());
}
}
synchronized int frequency(Query query) {
- return recentlyUsedFilters.frequency(QueryCache.cacheKey(query).hashCode());
+ return recentlyUsedFilters.frequency(cacheKey(query).hashCode());
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java Fri Feb 27 18:55:26 2015
@@ -327,6 +327,10 @@ public class TestOmitTf extends LuceneTe
new CountingHitCollector() {
private Scorer scorer;
@Override
+ public boolean needsScores() {
+ return true;
+ }
+ @Override
public final void setScorer(Scorer scorer) {
this.scorer = scorer;
}
@@ -345,6 +349,10 @@ public class TestOmitTf extends LuceneTe
new CountingHitCollector() {
private Scorer scorer;
@Override
+ public boolean needsScores() {
+ return true;
+ }
+ @Override
public final void setScorer(Scorer scorer) {
this.scorer = scorer;
}
@@ -366,6 +374,10 @@ public class TestOmitTf extends LuceneTe
new CountingHitCollector() {
private Scorer scorer;
@Override
+ public boolean needsScores() {
+ return true;
+ }
+ @Override
public final void setScorer(Scorer scorer) {
this.scorer = scorer;
}
@@ -385,6 +397,10 @@ public class TestOmitTf extends LuceneTe
new CountingHitCollector() {
private Scorer scorer;
@Override
+ public boolean needsScores() {
+ return true;
+ }
+ @Override
public final void setScorer(Scorer scorer) {
this.scorer = scorer;
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Fri Feb 27 18:55:26 2015
@@ -39,11 +39,24 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.RoaringDocIdSet;
public class TestCachingWrapperFilter extends LuceneTestCase {
+
+ private static FilterCachingPolicy MAYBE_CACHE_POLICY = new FilterCachingPolicy() {
+
+ @Override
+ public void onUse(Filter filter) {}
+
+ @Override
+ public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
+ return random().nextBoolean();
+ }
+
+ };
+
Directory dir;
DirectoryReader ir;
IndexSearcher is;
RandomIndexWriter iw;
-
+
@Override
public void setUp() throws Exception {
super.setUp();
@@ -64,7 +77,7 @@ public class TestCachingWrapperFilter ex
ir = iw.getReader();
is = newSearcher(ir);
}
-
+
@Override
public void tearDown() throws Exception {
iw.close();
@@ -84,7 +97,7 @@ public class TestCachingWrapperFilter ex
assertEquals(hits3.totalHits, hits4.totalHits);
CheckHits.checkEqual(query, hits3.scoreDocs, hits4.scoreDocs);
}
-
+
/** test null iterator */
public void testEmpty() throws Exception {
Query query = new BooleanQuery();
@@ -92,7 +105,7 @@ public class TestCachingWrapperFilter ex
Filter actual = new CachingWrapperFilter(expected, MAYBE_CACHE_POLICY);
assertFilterEquals(expected, actual);
}
-
+
/** test iterator returns NO_MORE_DOCS */
public void testEmpty2() throws Exception {
BooleanQuery query = new BooleanQuery();
@@ -102,14 +115,14 @@ public class TestCachingWrapperFilter ex
Filter actual = new CachingWrapperFilter(expected, MAYBE_CACHE_POLICY);
assertFilterEquals(expected, actual);
}
-
+
/** test null docidset */
public void testEmpty3() throws Exception {
Filter expected = new QueryWrapperFilter(new PrefixQuery(new Term("bogusField", "bogusVal")));
Filter actual = new CachingWrapperFilter(expected, MAYBE_CACHE_POLICY);
assertFilterEquals(expected, actual);
}
-
+
/** test iterator returns single document */
public void testSingle() throws Exception {
for (int i = 0; i < 10; i++) {
@@ -120,7 +133,7 @@ public class TestCachingWrapperFilter ex
assertFilterEquals(expected, actual);
}
}
-
+
/** test sparse filters (match single documents) */
public void testSparse() throws Exception {
for (int i = 0; i < 10; i++) {
@@ -133,7 +146,7 @@ public class TestCachingWrapperFilter ex
assertFilterEquals(expected, actual);
}
}
-
+
/** test dense filters (match entire index) */
public void testDense() throws Exception {
Query query = new MatchAllDocsQuery();
@@ -167,7 +180,7 @@ public class TestCachingWrapperFilter ex
reader.close();
dir.close();
}
-
+
public void testNullDocIdSet() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
@@ -190,11 +203,11 @@ public class TestCachingWrapperFilter ex
// the caching filter should return the empty set constant
assertNull(cacher.getDocIdSet(context, context.reader().getLiveDocs()));
-
+
reader.close();
dir.close();
}
-
+
public void testNullDocIdSetIterator() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
@@ -227,11 +240,11 @@ public class TestCachingWrapperFilter ex
// the caching filter should return the empty set constant
assertNull(cacher.getDocIdSet(context, context.reader().getLiveDocs()));
-
+
reader.close();
dir.close();
}
-
+
private static void assertDocIdSetCacheable(IndexReader reader, Filter filter, boolean shouldCacheable) throws IOException {
assertTrue(reader.getContext() instanceof LeafReaderContext);
LeafReaderContext context = (LeafReaderContext) reader.getContext();
@@ -254,7 +267,7 @@ public class TestCachingWrapperFilter ex
}
}
}
-
+
public void testIsCacheAble() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
@@ -397,7 +410,7 @@ public class TestCachingWrapperFilter ex
reader = refreshReader(reader);
searcher = newSearcher(reader, false);
-
+
docs = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), filter), 1);
assertEquals("[query + filter] Should find 2 hits...", 2, docs.totalHits);
assertTrue(filter.missCount > missCount);
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java Fri Feb 27 18:55:26 2015
@@ -23,10 +23,12 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
/** This class only tests some basic functionality in CSQ, the main parts are mostly
@@ -93,6 +95,7 @@ public class TestConstantScoreQuery exte
writer.close();
// we don't wrap with AssertingIndexSearcher in order to have the original scorer in setScorer.
searcher = newSearcher(reader, true, false);
+ searcher.setQueryCache(null); // to assert on scorer impl
// set a similarity that does not normalize our boost away
searcher.setSimilarity(new DefaultSimilarity() {
@@ -127,6 +130,27 @@ public class TestConstantScoreQuery exte
}
}
+ // a filter for which other queries don't have special rewrite rules
+ private static class FilterWrapper extends Filter {
+
+ private final Filter in;
+
+ FilterWrapper(Filter in) {
+ this.in = in;
+ }
+
+ @Override
+ public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
+ return in.getDocIdSet(context, acceptDocs);
+ }
+
+ @Override
+ public String toString(String field) {
+ return in.toString(field);
+ }
+
+ }
+
public void testConstantScoreQueryAndFilter() throws Exception {
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
@@ -139,13 +163,13 @@ public class TestConstantScoreQuery exte
IndexReader r = w.getReader();
w.close();
- Filter filterB = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "b"))));
+ Filter filterB = new FilterWrapper(new QueryWrapperFilter(new TermQuery(new Term("field", "b"))));
Query query = new ConstantScoreQuery(filterB);
IndexSearcher s = newSearcher(r);
assertEquals(1, s.search(new FilteredQuery(query, filterB), 1).totalHits); // Query for field:b, Filter field:b
- Filter filterA = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "a"))));
+ Filter filterA = new FilterWrapper(new QueryWrapperFilter(new TermQuery(new Term("field", "a"))));
query = new ConstantScoreQuery(filterA);
assertEquals(0, s.search(new FilteredQuery(query, filterB), 1).totalHits); // Query field:b, Filter field:a
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestDocValuesRangeQuery.java Fri Feb 27 18:55:26 2015
@@ -267,12 +267,12 @@ public class TestDocValuesRangeQuery ext
iw.close();
Query q1 = DocValuesRangeQuery.newLongRange("dv1", 0L, 100L, random().nextBoolean(), random().nextBoolean());
- Weight w = searcher.createNormalizedWeight(q1, random().nextBoolean());
+ Weight w = searcher.createNormalizedWeight(q1, true);
Scorer s = w.scorer(ctx, null);
assertNotNull(s.asTwoPhaseIterator());
Query q2 = DocValuesRangeQuery.newBytesRefRange("dv2", toSortableBytes(0L), toSortableBytes(100L), random().nextBoolean(), random().nextBoolean());
- w = searcher.createNormalizedWeight(q2, random().nextBoolean());
+ w = searcher.createNormalizedWeight(q2, true);
s = w.scorer(ctx, null);
assertNotNull(s.asTwoPhaseIterator());
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilterCachingPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilterCachingPolicy.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilterCachingPolicy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilterCachingPolicy.java Fri Feb 27 18:55:26 2015
@@ -38,11 +38,11 @@ public class TestFilterCachingPolicy ext
}
final IndexReader reader = w.getReader();
for (float minSizeRatio : new float[] {Float.MIN_VALUE, 0.01f, 0.1f, 0.9f}) {
- final FilterCachingPolicy policy = new FilterCachingPolicy.CacheOnLargeSegments(minSizeRatio);
+ final QueryCachingPolicy policy = new QueryCachingPolicy.CacheOnLargeSegments(minSizeRatio);
for (LeafReaderContext ctx : reader.leaves()) {
final Filter filter = new QueryWrapperFilter(new TermQuery(new Term("field", "value")));
final DocIdSet set = null;
- final boolean shouldCache = policy.shouldCache(filter, ctx, set);
+ final boolean shouldCache = policy.shouldCache(filter, ctx);
final float sizeRatio = (float) ctx.reader().maxDoc() / reader.maxDoc();
assertEquals(sizeRatio >= minSizeRatio, shouldCache);
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java Fri Feb 27 18:55:26 2015
@@ -302,17 +302,38 @@ public class TestFilteredQuery extends L
tChainedFilters(false);
}
+ // a filter for which other queries don't have special rewrite rules
+ private static class FilterWrapper extends Filter {
+
+ private final Filter in;
+
+ FilterWrapper(Filter in) {
+ this.in = in;
+ }
+
+ @Override
+ public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
+ return in.getDocIdSet(context, acceptDocs);
+ }
+
+ @Override
+ public String toString(String field) {
+ return in.toString(field);
+ }
+
+ }
+
private void tChainedFilters(final boolean useRandomAccess) throws Exception {
Query query = new FilteredQuery(new FilteredQuery(
- new MatchAllDocsQuery(), new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "three")))), randomFilterStrategy(random(), useRandomAccess)),
- new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "four")))), randomFilterStrategy(random(), useRandomAccess));
+ new MatchAllDocsQuery(), new FilterWrapper(new QueryWrapperFilter(new TermQuery(new Term("field", "three")))), randomFilterStrategy(random(), useRandomAccess)),
+ new FilterWrapper(new QueryWrapperFilter(new TermQuery(new Term("field", "four")))), randomFilterStrategy(random(), useRandomAccess));
ScoreDoc[] hits = searcher.search(query, 10).scoreDocs;
assertEquals(2, hits.length);
QueryUtils.check(random(), query, searcher);
// one more:
query = new FilteredQuery(query,
- new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "five")))), randomFilterStrategy(random(), useRandomAccess));
+ new FilterWrapper(new QueryWrapperFilter(new TermQuery(new Term("field", "five")))), randomFilterStrategy(random(), useRandomAccess));
hits = searcher.search(query, 10).scoreDocs;
assertEquals(1, hits.length);
QueryUtils.check(random(), query, searcher);
@@ -390,8 +411,8 @@ public class TestFilteredQuery extends L
}
public void testRewrite() throws Exception {
- assertRewrite(new FilteredQuery(new TermQuery(new Term("field", "one")), new CachingWrapperFilter(new QueryWrapperFilter(new PrefixQuery(new Term("field", "o")))), randomFilterStrategy()), FilteredQuery.class);
- assertRewrite(new FilteredQuery(new PrefixQuery(new Term("field", "one")), new CachingWrapperFilter(new QueryWrapperFilter(new PrefixQuery(new Term("field", "o")))), randomFilterStrategy()), FilteredQuery.class);
+ assertRewrite(new FilteredQuery(new TermQuery(new Term("field", "one")), new FilterWrapper(new QueryWrapperFilter(new PrefixQuery(new Term("field", "o")))), randomFilterStrategy()), FilteredQuery.class);
+ assertRewrite(new FilteredQuery(new PrefixQuery(new Term("field", "one")), new FilterWrapper(new QueryWrapperFilter(new PrefixQuery(new Term("field", "o")))), randomFilterStrategy()), FilteredQuery.class);
}
public void testGetFilterStrategy() {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUFilterCache.java Fri Feb 27 18:55:26 2015
@@ -34,6 +34,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
@@ -60,6 +61,18 @@ public class TestLRUFilterCache extends
};
+ private static FilterCachingPolicy MAYBE_CACHE_POLICY = new FilterCachingPolicy() {
+
+ @Override
+ public void onUse(Filter filter) {}
+
+ @Override
+ public boolean shouldCache(Filter filter, LeafReaderContext context, DocIdSet set) throws IOException {
+ return random().nextBoolean();
+ }
+
+ };
+
public void testFilterRamBytesUsed() {
final Filter simpleFilter = new QueryWrapperFilter(new TermQuery(new Term("some_field", "some_term")));
final long actualRamBytesUsed = RamUsageTester.sizeOf(simpleFilter);
@@ -73,7 +86,15 @@ public class TestLRUFilterCache extends
final LRUFilterCache filterCache = new LRUFilterCache(1 + random().nextInt(20), 1 + random().nextInt(10000));
Directory dir = newDirectory();
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
- final SearcherManager mgr = new SearcherManager(w.w, random().nextBoolean(), new SearcherFactory());
+ final SearcherManager mgr = new SearcherManager(w.w, random().nextBoolean(), new SearcherFactory() {
+ @Override
+ public IndexSearcher newSearcher(IndexReader reader) throws IOException {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ // disable built-in caching
+ searcher.setQueryCache(null);
+ return searcher;
+ }
+ });
final AtomicBoolean indexing = new AtomicBoolean(true);
final AtomicReference<Throwable> error = new AtomicReference<>();
final int numDocs = atLeast(10000);
@@ -164,6 +185,7 @@ public class TestLRUFilterCache extends
w.addDocument(doc);
final DirectoryReader reader = w.getReader();
final IndexSearcher searcher = newSearcher(reader);
+ searcher.setQueryCache(null);
final LRUFilterCache filterCache = new LRUFilterCache(2, 100000);
final Filter blue = new QueryWrapperFilter(new TermQuery(new Term("color", "blue")));
@@ -329,6 +351,7 @@ public class TestLRUFilterCache extends
}
try (final DirectoryReader reader = w.getReader()) {
final IndexSearcher searcher = new IndexSearcher(reader);
+ searcher.setQueryCache(null);
for (int i = 0; i < 3; ++i) {
final Filter filter = new QueryWrapperFilter(new TermQuery(new Term("color", RandomPicks.randomFrom(random(), colors))));
searcher.search(new ConstantScoreQuery(filterCache.doCache(filter, MAYBE_CACHE_POLICY)), 1);
@@ -390,6 +413,7 @@ public class TestLRUFilterCache extends
}
final DirectoryReader reader = w.getReader();
final IndexSearcher searcher = new IndexSearcher(reader);
+ searcher.setQueryCache(null);
final int numFilters = atLeast(1000);
for (int i = 0; i < numFilters; ++i) {
@@ -427,6 +451,7 @@ public class TestLRUFilterCache extends
}
final DirectoryReader reader = w.getReader();
final IndexSearcher searcher = new IndexSearcher(reader);
+ searcher.setQueryCache(null);
final Map<Filter, Integer> actualCounts = new HashMap<>();
final Map<Filter, Integer> expectedCounts = new HashMap<>();
@@ -486,6 +511,7 @@ public class TestLRUFilterCache extends
final DirectoryReader reader = w.getReader();
final int segmentCount = reader.leaves().size();
final IndexSearcher searcher = new IndexSearcher(reader);
+ searcher.setQueryCache(null);
final Filter filter = new QueryWrapperFilter(new TermQuery(new Term("color", "red")));
final Filter filter2 = new QueryWrapperFilter(new TermQuery(new Term("color", "blue")));
@@ -574,10 +600,12 @@ public class TestLRUFilterCache extends
final DirectoryReader reader1 = w1.getReader();
final int segmentCount1 = reader1.leaves().size();
final IndexSearcher searcher1 = new IndexSearcher(reader1);
+ searcher1.setQueryCache(null);
final DirectoryReader reader2 = w2.getReader();
final int segmentCount2 = reader2.leaves().size();
final IndexSearcher searcher2 = new IndexSearcher(reader2);
+ searcher2.setQueryCache(null);
final Map<Object, Integer> indexId = new HashMap<>();
for (LeafReaderContext ctx : reader1.leaves()) {
Copied: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java (from r1662774, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java?p2=lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java&r1=1662774&r2=1662791&rev=1662791&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java Fri Feb 27 18:55:26 2015
@@ -734,6 +734,17 @@ public class TestLRUQueryCache extends L
dir2.close();
}
+ private static Query cacheKey(Query query) {
+ if (query.getBoost() == 1f) {
+ return query;
+ } else {
+ Query key = query.clone();
+ key.setBoost(1f);
+ assert key == cacheKey(key);
+ return key;
+ }
+ }
+
public void testUseRewrittenQueryAsCacheKey() throws IOException {
final Query expectedCacheKey = new TermQuery(new Term("foo", "bar"));
final BooleanQuery query = new BooleanQuery();
@@ -756,13 +767,13 @@ public class TestLRUQueryCache extends L
@Override
public boolean shouldCache(Query query, LeafReaderContext context) throws IOException {
- assertEquals(expectedCacheKey, QueryCache.cacheKey(query));
+ assertEquals(expectedCacheKey, cacheKey(query));
return true;
}
@Override
public void onUse(Query query) {
- assertEquals(expectedCacheKey, QueryCache.cacheKey(query));
+ assertEquals(expectedCacheKey, cacheKey(query));
}
};
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPrefixRandom.java Fri Feb 27 18:55:26 2015
@@ -107,6 +107,15 @@ public class TestPrefixRandom extends Lu
public String toString(String field) {
return field.toString() + ":" + prefix.toString();
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (super.equals(obj) == false) {
+ return false;
+ }
+ final DumbPrefixQuery that = (DumbPrefixQuery) obj;
+ return prefix.equals(that.prefix);
+ }
}
/** test a bunch of random prefixes */
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestQueryWrapperFilter.java Fri Feb 27 18:55:26 2015
@@ -25,15 +25,38 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.English;
import org.apache.lucene.util.LuceneTestCase;
public class TestQueryWrapperFilter extends LuceneTestCase {
+ // a filter for which other queries don't have special rewrite rules
+ private static class FilterWrapper extends Filter {
+
+ private final Filter in;
+
+ FilterWrapper(Filter in) {
+ this.in = in;
+ }
+
+ @Override
+ public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
+ return in.getDocIdSet(context, acceptDocs);
+ }
+
+ @Override
+ public String toString(String field) {
+ return in.toString(field);
+ }
+
+ }
+
public void testBasic() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
@@ -51,7 +74,7 @@ public class TestQueryWrapperFilter exte
IndexSearcher searcher = newSearcher(reader);
TopDocs hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), qwf), 10);
assertEquals(1, hits.totalHits);
- hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), new CachingWrapperFilter(qwf)), 10);
+ hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), new FilterWrapper(qwf)), 10);
assertEquals(1, hits.totalHits);
// should not throw exception with complex primitive query
@@ -63,7 +86,7 @@ public class TestQueryWrapperFilter exte
hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), qwf), 10);
assertEquals(1, hits.totalHits);
- hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), new CachingWrapperFilter(qwf)), 10);
+ hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), new FilterWrapper(qwf)), 10);
assertEquals(1, hits.totalHits);
// should not throw exception with non primitive Query (doesn't implement
@@ -72,7 +95,7 @@ public class TestQueryWrapperFilter exte
hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), qwf), 10);
assertEquals(1, hits.totalHits);
- hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), new CachingWrapperFilter(qwf)), 10);
+ hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), new FilterWrapper(qwf)), 10);
assertEquals(1, hits.totalHits);
// test a query with no hits
@@ -80,7 +103,7 @@ public class TestQueryWrapperFilter exte
qwf = new QueryWrapperFilter(termQuery);
hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), qwf), 10);
assertEquals(0, hits.totalHits);
- hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), new CachingWrapperFilter(qwf)), 10);
+ hits = searcher.search(new FilteredQuery(new MatchAllDocsQuery(), new FilterWrapper(qwf)), 10);
assertEquals(0, hits.totalHits);
reader.close();
dir.close();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestRegexpRandom2.java Fri Feb 27 18:55:26 2015
@@ -138,6 +138,15 @@ public class TestRegexpRandom2 extends L
public String toString(String field) {
return field.toString() + automaton.toString();
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (super.equals(obj) == false) {
+ return false;
+ }
+ final DumbRegexpQuery that = (DumbRegexpQuery) obj;
+ return automaton.equals(that.automaton);
+ }
}
/** test a bunch of random regular expressions */
@@ -146,7 +155,7 @@ public class TestRegexpRandom2 extends L
for (int i = 0; i < num; i++) {
String reg = AutomatonTestUtil.randomRegexp(random());
if (VERBOSE) {
- System.out.println("TEST: regexp=" + reg);
+ System.out.println("TEST: regexp='" + reg + "'");
}
assertSame(reg);
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestUsageTrackingFilterCachingPolicy.java Fri Feb 27 18:55:26 2015
@@ -19,26 +19,26 @@ package org.apache.lucene.search;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.RoaringDocIdSet;
public class TestUsageTrackingFilterCachingPolicy extends LuceneTestCase {
- public void testCheapToCache() {
- assertTrue(UsageTrackingFilterCachingPolicy.isCheapToCache(null));
- assertTrue(UsageTrackingFilterCachingPolicy.isCheapToCache(DocIdSet.EMPTY));
- assertTrue(UsageTrackingFilterCachingPolicy.isCheapToCache(new RoaringDocIdSet.Builder(5).add(3).build()));
- assertFalse(UsageTrackingFilterCachingPolicy.isCheapToCache(new DocValuesDocIdSet(5, null) {
- @Override
- protected boolean matchDoc(int doc) {
- return false;
- }
- }));
+ public void testCostlyFilter() {
+ assertTrue(UsageTrackingQueryCachingPolicy.isCostly(new PrefixQuery(new Term("field", "prefix"))));
+ assertTrue(UsageTrackingQueryCachingPolicy.isCostly(NumericRangeQuery.newIntRange("intField", 8, 1, 1000, true, true)));
+ assertFalse(UsageTrackingQueryCachingPolicy.isCostly(new TermQuery(new Term("field", "value"))));
}
- public void testCostlyFilter() {
- assertTrue(UsageTrackingFilterCachingPolicy.isCostly(new QueryWrapperFilter(new PrefixQuery(new Term("field", "prefix")))));
- assertTrue(UsageTrackingFilterCachingPolicy.isCostly(new QueryWrapperFilter(NumericRangeQuery.newIntRange("intField", 8, 1, 1000, true, true))));
- assertFalse(UsageTrackingFilterCachingPolicy.isCostly(new QueryWrapperFilter(new TermQuery(new Term("field", "value")))));
+ public void testBoostIgnored() {
+ Query q1 = new TermQuery(new Term("foo", "bar"));
+ q1.setBoost(2);
+ Query q2 = q1.clone();
+ q2.setBoost(3);
+ Query q3 = q1.clone();
+ q3.setBoost(4);
+ UsageTrackingQueryCachingPolicy policy = new UsageTrackingQueryCachingPolicy();
+ policy.onUse(q1);
+ policy.onUse(q2);
+ assertEquals(2, policy.frequency(q3));
}
}
Modified: lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java (original)
+++ lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSideways.java Fri Feb 27 18:55:26 2015
@@ -26,6 +26,7 @@ import org.apache.lucene.facet.sortedset
import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
import org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.search.FilterCollector;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@@ -173,6 +174,16 @@ public class DrillSideways {
drillDownQueries[i-startClause] = clauses[i].getQuery();
}
DrillSidewaysQuery dsq = new DrillSidewaysQuery(baseQuery, drillDownCollector, drillSidewaysCollectors, drillDownQueries, scoreSubDocsAtOnce());
+ if (hitCollector.needsScores() == false) {
+ // this is a borrible hack in order to make sure IndexSearcher will not
+ // attempt to cache the DrillSidewaysQuery
+ hitCollector = new FilterCollector(hitCollector) {
+ @Override
+ public boolean needsScores() {
+ return true;
+ }
+ };
+ }
searcher.search(dsq, hitCollector);
return new DrillSidewaysResult(buildFacetsResult(drillDownCollector, drillSidewaysCollectors, drillDownDims.keySet().toArray(new String[drillDownDims.size()])), null);
Modified: lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java Fri Feb 27 18:55:26 2015
@@ -33,8 +33,10 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
/** Only purpose is to punch through and return a
- * DrillSidewaysScorer */
+ * DrillSidewaysScorer*/
+// TODO change the way DrillSidewaysScorer is used, this query does not work
+// with filter caching
class DrillSidewaysQuery extends Query {
final Query baseQuery;
final Collector drillDownCollector;
Modified: lucene/dev/branches/branch_5x/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java (original)
+++ lucene/dev/branches/branch_5x/lucene/facet/src/test/org/apache/lucene/facet/range/TestRangeFacetCounts.java Fri Feb 27 18:55:26 2015
@@ -55,17 +55,18 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
-import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.CachingWrapperQuery;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.FilterCachingPolicy;
+import org.apache.lucene.search.QueryCachingPolicy;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitDocIdSet;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
@@ -908,16 +909,19 @@ public class TestRangeFacetCounts extend
final AtomicBoolean filterWasUsed = new AtomicBoolean();
if (random().nextBoolean()) {
// Sort of silly:
- fastMatchFilter = new CachingWrapperFilter(new QueryWrapperFilter(new MatchAllDocsQuery()), FilterCachingPolicy.ALWAYS_CACHE) {
- @Override
- protected DocIdSet cacheImpl(DocIdSetIterator iterator, LeafReader reader)
+ final Filter in = new QueryWrapperFilter(new MatchAllDocsQuery());
+ fastMatchFilter = new Filter() {
+ @Override
+ public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs)
throws IOException {
- final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
- filterWasUsed.set(true);
- cached.or(iterator);
- return new BitDocIdSet(cached);
- }
- };
+ filterWasUsed.set(true);
+ return in.getDocIdSet(context, acceptDocs);
+ }
+ @Override
+ public String toString(String field) {
+ return in.toString(field);
+ }
+ };
} else {
fastMatchFilter = null;
}
Modified: lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java (original)
+++ lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/GroupingSearchTest.java Fri Feb 27 18:55:26 2015
@@ -28,7 +28,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
-import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.CachingWrapperQuery;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryWrapperFilter;
@@ -159,7 +159,7 @@ public class GroupingSearchTest extends
assertEquals(1, group.scoreDocs.length);
assertEquals(6, group.scoreDocs[0].doc);
- Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
+ Filter lastDocInBlock = new QueryWrapperFilter(new TermQuery(new Term("groupend", "x")));
groupingSearch = new GroupingSearch(lastDocInBlock);
groups = groupingSearch.search(indexSearcher, new TermQuery(new Term("content", "random")), 0, 10);
Modified: lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (original)
+++ lucene/dev/branches/branch_5x/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java Fri Feb 27 18:55:26 2015
@@ -51,7 +51,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.search.CachingCollector;
-import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.CachingWrapperQuery;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Filter;
@@ -796,7 +796,7 @@ public class TestGrouping extends Lucene
// group, so we can use single pass collector
dirBlocks = newDirectory();
rBlocks = getDocBlockReader(dirBlocks, groupDocs);
- final Filter lastDocInBlock = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("groupend", "x"))));
+ final Filter lastDocInBlock = new QueryWrapperFilter(new TermQuery(new Term("groupend", "x")));
final NumericDocValues docIDToIDBlocks = MultiDocValues.getNumericValues(rBlocks, "id");
assertNotNull(docIDToIDBlocks);
Modified: lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/BitDocIdSetCachingWrapperFilter.java Fri Feb 27 18:55:26 2015
@@ -18,81 +18,82 @@ package org.apache.lucene.search.join;
*/
import java.io.IOException;
-import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.WeakHashMap;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.FilterCachingPolicy;
-import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BitDocIdSet;
/**
- * A filter wrapper that transforms the produces doc id sets into
- * {@link BitDocIdSet}s if necessary and caches them.
+ * {@link Filter} wrapper that implements {@link BitDocIdSetFilter}.
*/
-public class BitDocIdSetCachingWrapperFilter extends BitDocIdSetFilter implements Accountable {
-
- private final CachingWrapperFilter filter;
-
- /** Sole constructor. */
+public class BitDocIdSetCachingWrapperFilter extends BitDocIdSetFilter {
+ private final Filter filter;
+ private final Map<Object,DocIdSet> cache = Collections.synchronizedMap(new WeakHashMap<Object,DocIdSet>());
+
+ /** Wraps another filter's result and caches it into bitsets.
+ * @param filter Filter to cache results of
+ */
public BitDocIdSetCachingWrapperFilter(Filter filter) {
- super();
- this.filter = new CachingWrapperFilter(filter, FilterCachingPolicy.ALWAYS_CACHE) {
- @Override
- protected BitDocIdSet docIdSetToCache(DocIdSet docIdSet, LeafReader reader) throws IOException {
- if (docIdSet == null || docIdSet instanceof BitDocIdSet) {
- // this is different from CachingWrapperFilter: even when the DocIdSet is
- // cacheable, we convert it to a BitSet since we require all the
- // cached filters to be BitSets
- return (BitDocIdSet) docIdSet;
- }
-
- final DocIdSetIterator it = docIdSet.iterator();
- if (it == null) {
- return null;
- }
- BitDocIdSet.Builder builder = new BitDocIdSet.Builder(reader.maxDoc());
- builder.or(it);
- return builder.build();
- }
- };
+ this.filter = filter;
}
- @Override
- public BitDocIdSet getDocIdSet(LeafReaderContext context) throws IOException {
- return (BitDocIdSet) filter.getDocIdSet(context, null);
+ /**
+ * Gets the contained filter.
+ * @return the contained filter.
+ */
+ public Filter getFilter() {
+ return filter;
+ }
+
+ private BitDocIdSet docIdSetToCache(DocIdSet docIdSet, LeafReader reader) throws IOException {
+ final DocIdSetIterator it = docIdSet.iterator();
+ if (it == null) {
+ return null;
+ } else {
+ BitDocIdSet.Builder builder = new BitDocIdSet.Builder(reader.maxDoc());
+ builder.or(it);
+ return builder.build();
+ }
}
-
+
@Override
- public int hashCode() {
- return getClass().hashCode() ^ filter.hashCode();
- }
+ public BitDocIdSet getDocIdSet(LeafReaderContext context) throws IOException {
+ final LeafReader reader = context.reader();
+ final Object key = reader.getCoreCacheKey();
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof BitDocIdSetCachingWrapperFilter == false) {
- return false;
+ DocIdSet docIdSet = cache.get(key);
+ if (docIdSet == null) {
+ docIdSet = filter.getDocIdSet(context, null);
+ docIdSet = docIdSetToCache(docIdSet, reader);
+ if (docIdSet == null) {
+ // We use EMPTY as a sentinel for the empty set, which is cacheable
+ docIdSet = DocIdSet.EMPTY;
+ }
+ cache.put(key, docIdSet);
}
- return filter.equals(((BitDocIdSetCachingWrapperFilter) obj).filter);
+ return docIdSet == DocIdSet.EMPTY ? null : (BitDocIdSet) docIdSet;
}
-
+
@Override
public String toString(String field) {
- return filter.toString();
+ return getClass().getSimpleName() + "("+filter.toString(field)+")";
}
@Override
- public long ramBytesUsed() {
- return filter.ramBytesUsed();
+ public boolean equals(Object o) {
+ if (o == null || !getClass().equals(o.getClass())) return false;
+ final BitDocIdSetCachingWrapperFilter other = (BitDocIdSetCachingWrapperFilter) o;
+ return this.filter.equals(other.filter);
}
@Override
- public Collection<Accountable> getChildResources() {
- return filter.getChildResources();
+ public int hashCode() {
+ return (filter.hashCode() ^ getClass().hashCode());
}
-
}
Modified: lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java (original)
+++ lucene/dev/branches/branch_5x/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java Fri Feb 27 18:55:26 2015
@@ -501,6 +501,8 @@ public class ToParentBlockJoinCollector
@Override
public boolean needsScores() {
- return trackScores || trackMaxScore || sort.needsScores();
+ // needed so that eg. BooleanQuery does not rewrite its MUST clauses to
+ // FILTER since the filter scorers are hidden in Scorer.getChildren().
+ return true;
}
}
Modified: lucene/dev/branches/branch_5x/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java (original)
+++ lucene/dev/branches/branch_5x/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoinValidation.java Fri Feb 27 18:55:26 2015
@@ -17,6 +17,7 @@ package org.apache.lucene.search.join;
* limitations under the License.
*/
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -27,10 +28,12 @@ import org.apache.lucene.index.Directory
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.CachingWrapperFilter;
+import org.apache.lucene.search.CachingWrapperQuery;
+import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.IndexSearcher;
@@ -39,6 +42,7 @@ import org.apache.lucene.search.QueryWra
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.After;
import org.junit.Before;
@@ -115,12 +119,33 @@ public class TestBlockJoinValidation ext
indexSearcher.search(blockJoinQuery, 1);
}
+ // a filter for which other queries don't have special rewrite rules
+ private static class FilterWrapper extends Filter {
+
+ private final Filter in;
+
+ FilterWrapper(Filter in) {
+ this.in = in;
+ }
+
+ @Override
+ public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
+ return in.getDocIdSet(context, acceptDocs);
+ }
+
+ @Override
+ public String toString(String field) {
+ return in.toString(field);
+ }
+
+ }
+
@Test
public void testValidationForToChildBjqWithChildFilterQuery() throws Exception {
Query parentQueryWithRandomChild = createParentQuery();
ToChildBlockJoinQuery blockJoinQuery = new ToChildBlockJoinQuery(parentQueryWithRandomChild, parentsFilter);
- Filter childFilter = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("common_field", "1"))));
+ Filter childFilter = new FilterWrapper(new QueryWrapperFilter(new TermQuery(new Term("common_field", "1"))));
thrown.expect(IllegalStateException.class);
thrown.expectMessage(ToChildBlockJoinQuery.ILLEGAL_ADVANCE_ON_PARENT);
indexSearcher.search(new FilteredQuery(blockJoinQuery, childFilter), 1);
@@ -137,8 +162,8 @@ public class TestBlockJoinValidation ext
// advance() method is used by ConjunctionScorer, so we need to create Boolean conjunction query
BooleanQuery conjunctionQuery = new BooleanQuery();
WildcardQuery childQuery = new WildcardQuery(new Term("child", createFieldValue(randomChildNumber)));
- conjunctionQuery.add(new BooleanClause(childQuery, BooleanClause.Occur.MUST));
- conjunctionQuery.add(new BooleanClause(blockJoinQuery, BooleanClause.Occur.MUST));
+ conjunctionQuery.add(childQuery, BooleanClause.Occur.MUST);
+ conjunctionQuery.add(blockJoinQuery, BooleanClause.Occur.MUST);
thrown.expect(IllegalStateException.class);
thrown.expectMessage(ToChildBlockJoinQuery.INVALID_QUERY_MESSAGE);
Modified: lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestBlockJoinSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestBlockJoinSorter.java?rev=1662791&r1=1662790&r2=1662791&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestBlockJoinSorter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/misc/src/test/org/apache/lucene/index/TestBlockJoinSorter.java Fri Feb 27 18:55:26 2015
@@ -17,9 +17,14 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import static org.apache.lucene.search.DocIdSet.EMPTY;
+
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -32,37 +37,60 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.BlockJoinComparatorSource;
-import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.FilterCachingPolicy;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitDocIdSet;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SparseFixedBitSet;
public class TestBlockJoinSorter extends LuceneTestCase {
- private static class FixedBitSetCachingWrapperFilter extends CachingWrapperFilter {
+ private static class BitSetCachingWrapperFilter extends Filter {
+
+ private final Filter filter;
+ private final Map<Object,BitDocIdSet> cache = Collections.synchronizedMap(new WeakHashMap<Object,BitDocIdSet>());
- public FixedBitSetCachingWrapperFilter(Filter filter) {
- super(filter, FilterCachingPolicy.ALWAYS_CACHE);
+ public BitSetCachingWrapperFilter(Filter filter) {
+ this.filter = filter;
}
@Override
- protected DocIdSet cacheImpl(DocIdSetIterator iterator, LeafReader reader)
- throws IOException {
- final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
- cached.or(iterator);
- return new BitDocIdSet(cached);
+ public DocIdSet getDocIdSet(LeafReaderContext context, final Bits acceptDocs) throws IOException {
+ final LeafReader reader = context.reader();
+ final Object key = reader.getCoreCacheKey();
+
+ BitDocIdSet docIdSet = cache.get(key);
+ if (docIdSet == null) {
+ final DocIdSet uncached = filter.getDocIdSet(context, null);
+ final DocIdSetIterator it = uncached == null ? null : uncached.iterator();
+ if (it != null) {
+ BitDocIdSet.Builder builder = new BitDocIdSet.Builder(context.reader().maxDoc());
+ builder.or(it);
+ docIdSet = builder.build();
+ }
+ if (docIdSet == null) {
+ docIdSet = new BitDocIdSet(new SparseFixedBitSet(context.reader().maxDoc()));
+ }
+ cache.put(key, docIdSet);
+ }
+
+ return docIdSet == EMPTY ? null : BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
}
+ @Override
+ public String toString(String field) {
+ return getClass().getName() + "(" + filter.toString(field) + ")";
+ }
}
public void test() throws IOException {
@@ -92,7 +120,7 @@ public class TestBlockJoinSorter extends
writer.close();
final LeafReader reader = getOnlySegmentReader(indexReader);
- final Filter parentsFilter = new FixedBitSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("parent", "true"))));
+ final Filter parentsFilter = new BitSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("parent", "true"))));
final FixedBitSet parentBits = (FixedBitSet) parentsFilter.getDocIdSet(reader.getContext(), null).bits();
final NumericDocValues parentValues = reader.getNumericDocValues("parent_val");
final NumericDocValues childValues = reader.getNumericDocValues("child_val");