You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2015/06/25 15:19:11 UTC
svn commit: r1687514 - in /lucene/dev/branches/branch_5x/lucene: ./
core/src/java/org/apache/lucene/search/
core/src/test/org/apache/lucene/search/
test-framework/src/java/org/apache/lucene/search/
Author: jpountz
Date: Thu Jun 25 13:19:10 2015
New Revision: 1687514
URL: http://svn.apache.org/r1687514
Log:
LUCENE-6601: Make FilteredQuery.FilterStrategy leverage the two-phase iteration API instead of the acceptDocs parameter of the Scorer API.
Modified:
lucene/dev/branches/branch_5x/lucene/CHANGES.txt
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/Filter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1687514&r1=1687513&r2=1687514&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Thu Jun 25 13:19:10 2015
@@ -92,6 +92,10 @@ API Changes
SpanPayloadNearCheckQuery have moved from the .spans package to the .payloads
package. (Alan Woodward, David Smiley, Paul Elschot, Robert Muir)
+* LUCENE-6583: FilteredQuery is deprecated and will be removed in 6.0. It should
+ be replaced with a BooleanQuery which handle the query as a MUST clause and
+ the filter as a FILTER clause. (Adrien Grand)
+
Bug fixes
* LUCENE-6500: ParallelCompositeReader did not always call
@@ -158,6 +162,15 @@ Changes in Runtime Behavior
* LUCENE-2880: Span queries now score more consistently with regular queries.
(Robert Muir, Adrien Grand)
+* LUCENE-6601: FilteredQuery now always rewrites to a BooleanQuery which handles
+ the query as a MUST clause and the filter as a FILTER clause.
+ LEAP_FROG_QUERY_FIRST_STRATEGY and LEAP_FROG_FILTER_FIRST_STRATEGY do not
+ guarantee anymore which iterator will be advanced first, it will depend on the
+ respective costs of the iterators. QUERY_FIRST_FILTER_STRATEGY and
+ RANDOM_ACCESS_FILTER_STRATEGY still consume the filter using its random-access
+ API, however the returned bits may be called on different documents compared
+ to before. (Adrien Grand)
+
Optimizations
* LUCENE-6548: Some optimizations for BlockTree's intersect with very
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/Filter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/Filter.java?rev=1687514&r1=1687513&r2=1687514&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/Filter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/Filter.java Thu Jun 25 13:19:10 2015
@@ -18,10 +18,9 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.util.Set;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.Term;
import org.apache.lucene.util.Bits;
/**
@@ -64,45 +63,7 @@ public abstract class Filter extends Que
//
@Override
- public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
- return new Weight(this) {
-
- @Override
- public void extractTerms(Set<Term> terms) {}
-
- @Override
- public float getValueForNormalization() throws IOException {
- return 0f;
- }
-
- @Override
- public void normalize(float norm, float topLevelBoost) {}
-
- @Override
- public Explanation explain(LeafReaderContext context, int doc) throws IOException {
- final Scorer scorer = scorer(context, context.reader().getLiveDocs());
- final boolean match = (scorer != null && scorer.advance(doc) == doc);
- if (match) {
- assert scorer.score() == 0f;
- return Explanation.match(0f, "Match on id " + doc);
- } else {
- return Explanation.match(0f, "No match on id " + doc);
- }
- }
-
- @Override
- public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
- final DocIdSet set = getDocIdSet(context, acceptDocs);
- if (set == null) {
- return null;
- }
- final DocIdSetIterator iterator = set.iterator();
- if (iterator == null) {
- return null;
- }
- return new ConstantScoreScorer(this, 0f, iterator);
- }
-
- };
+ public Query rewrite(IndexReader reader) throws IOException {
+ return FilteredQuery.RANDOM_ACCESS_FILTER_STRATEGY.rewrite(this);
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1687514&r1=1687513&r2=1687514&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java Thu Jun 25 13:19:10 2015
@@ -18,9 +18,6 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
import java.util.Objects;
import java.util.Set;
@@ -77,282 +74,14 @@ public class FilteredQuery extends Query
this.filter = Objects.requireNonNull(filter, "Filter must not be null");
}
- /**
- * Returns a Weight that applies the filter to the enclosed query's Weight.
- * This is accomplished by overriding the Scorer returned by the Weight.
- */
- @Override
- public Weight createWeight(final IndexSearcher searcher, boolean needsScores) throws IOException {
- final Weight weight = query.createWeight (searcher, needsScores);
- return new Weight(FilteredQuery.this) {
-
- @Override
- public void extractTerms(Set<Term> terms) {
- weight.extractTerms(terms);
- }
-
- @Override
- public float getValueForNormalization() throws IOException {
- return weight.getValueForNormalization() * getBoost() * getBoost(); // boost sub-weight
- }
-
- @Override
- public void normalize(float norm, float topLevelBoost) {
- weight.normalize(norm, topLevelBoost * getBoost()); // incorporate boost
- }
-
- @Override
- public Explanation explain(LeafReaderContext ir, int i) throws IOException {
- Explanation inner = weight.explain (ir, i);
- Filter f = FilteredQuery.this.filter;
- DocIdSet docIdSet = f.getDocIdSet(ir, ir.reader().getLiveDocs());
- DocIdSetIterator docIdSetIterator = docIdSet == null ? DocIdSetIterator.empty() : docIdSet.iterator();
- if (docIdSetIterator == null) {
- docIdSetIterator = DocIdSetIterator.empty();
- }
- if (docIdSetIterator.advance(i) == i) {
- return inner;
- } else {
- return Explanation.noMatch("failure to match filter: " + f.toString(), inner);
- }
- }
-
- // return a filtering scorer
- @Override
- public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
- assert filter != null;
-
- DocIdSet filterDocIdSet = filter.getDocIdSet(context, acceptDocs);
- if (filterDocIdSet == null) {
- // this means the filter does not accept any documents.
- return null;
- }
-
- return strategy.filteredScorer(context, weight, filterDocIdSet);
- }
-
- // return a filtering top scorer
- @Override
- public BulkScorer bulkScorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
- assert filter != null;
-
- DocIdSet filterDocIdSet = filter.getDocIdSet(context, acceptDocs);
- if (filterDocIdSet == null) {
- // this means the filter does not accept any documents.
- return null;
- }
-
- return strategy.filteredBulkScorer(context, weight, filterDocIdSet);
-
- }
- };
- }
-
- /**
- * A scorer that consults the filter iff a document was matched by the
- * delegate scorer. This is useful if the filter computation is more expensive
- * than document scoring or if the filter has a linear running time to compute
- * the next matching doc like exact geo distances.
- */
- private static final class QueryFirstScorer extends Scorer {
- private final Scorer scorer;
- private final Bits filterBits;
-
- protected QueryFirstScorer(Weight weight, Bits filterBits, Scorer other) {
- super(weight);
- this.scorer = other;
- this.filterBits = filterBits;
- }
-
- @Override
- public int nextDoc() throws IOException {
- int doc;
- for(;;) {
- doc = scorer.nextDoc();
- if (doc == DocIdSetIterator.NO_MORE_DOCS || filterBits.get(doc)) {
- return doc;
- }
- }
- }
-
- @Override
- public int advance(int target) throws IOException {
- int doc = scorer.advance(target);
- if (doc != DocIdSetIterator.NO_MORE_DOCS && !filterBits.get(doc)) {
- return nextDoc();
- } else {
- return doc;
- }
- }
-
- @Override
- public int docID() {
- return scorer.docID();
- }
-
- @Override
- public float score() throws IOException {
- return scorer.score();
- }
-
- @Override
- public int freq() throws IOException {
- return scorer.freq();
- }
-
- @Override
- public long cost() {
- return scorer.cost();
- }
-
- @Override
- public Collection<ChildScorer> getChildren() {
- return Collections.singleton(new ChildScorer(scorer, "FILTERED"));
- }
-
- @Override
- public TwoPhaseIterator asTwoPhaseIterator() {
- final TwoPhaseIterator inner = scorer.asTwoPhaseIterator();
- if (inner != null) {
- // we are like a simplified conjunction here, handle the nested case:
- return new TwoPhaseIterator(inner.approximation()) {
- @Override
- public boolean matches() throws IOException {
- // check the approximation matches first, then check bits last.
- return inner.matches() && filterBits.get(scorer.docID());
- }
- };
- } else {
- // scorer doesnt have an approximation, just use it, to force bits applied last.
- return new TwoPhaseIterator(scorer) {
- @Override
- public boolean matches() throws IOException {
- return filterBits.get(scorer.docID());
- }
- };
- }
- }
- }
-
- private static class QueryFirstBulkScorer extends BulkScorer {
-
- private final Scorer scorer;
- private final Bits filterBits;
-
- public QueryFirstBulkScorer(Scorer scorer, Bits filterBits) {
- this.scorer = scorer;
- this.filterBits = filterBits;
- }
-
- @Override
- public long cost() {
- return scorer.cost();
- }
-
- @Override
- public int score(LeafCollector collector, int min, int maxDoc) throws IOException {
- // the normalization trick already applies the boost of this query,
- // so we can use the wrapped scorer directly:
- collector.setScorer(scorer);
- if (scorer.docID() < min) {
- scorer.advance(min);
- }
- while (true) {
- final int scorerDoc = scorer.docID();
- if (scorerDoc < maxDoc) {
- if (filterBits.get(scorerDoc)) {
- collector.collect(scorerDoc);
- }
- scorer.nextDoc();
- } else {
- break;
- }
- }
-
- return scorer.docID();
- }
- }
-
- /**
- * A Scorer that uses a "leap-frog" approach (also called "zig-zag join"). The scorer and the filter
- * take turns trying to advance to each other's next matching document, often
- * jumping past the target document. When both land on the same document, it's
- * collected.
- */
- private static final class LeapFrogScorer extends Scorer {
- private final ConjunctionDISI conjunction;
- private final Scorer scorer;
-
- protected LeapFrogScorer(Weight weight, DocIdSetIterator primary, DocIdSetIterator secondary, Scorer scorer) {
- super(weight);
- conjunction = ConjunctionDISI.intersect(Arrays.asList(primary, secondary));
- this.scorer = scorer;
- }
-
- @Override
- public int nextDoc() throws IOException {
- return conjunction.nextDoc();
- }
-
- @Override
- public final int advance(int target) throws IOException {
- return conjunction.advance(target);
- }
-
- @Override
- public final int docID() {
- return conjunction.docID();
- }
-
- @Override
- public final Collection<ChildScorer> getChildren() {
- return Collections.singleton(new ChildScorer(scorer, "FILTERED"));
- }
-
- @Override
- public long cost() {
- return conjunction.cost();
- }
-
- @Override
- public float score() throws IOException {
- return scorer.score();
- }
-
- @Override
- public int freq() throws IOException {
- return scorer.freq();
- }
-
- @Override
- public TwoPhaseIterator asTwoPhaseIterator() {
- return conjunction.asTwoPhaseIterator();
- }
- }
@Override
public Query rewrite(IndexReader reader) throws IOException {
- final Query queryRewritten = query.rewrite(reader);
- final Query filterRewritten = filter.rewrite(reader);
-
- if (queryRewritten != query || filterRewritten != filter) {
- // rewrite to a new FilteredQuery wrapping the rewritten query/filter
- if (filterRewritten instanceof Filter) {
- final Query rewritten = new FilteredQuery(queryRewritten, (Filter) filterRewritten, strategy);
- rewritten.setBoost(this.getBoost());
- return rewritten;
- } else {
- // In that case the filter does not implement random-access anyway so
- // we want to take advantage of approximations
- BooleanQuery rewritten = new BooleanQuery();
- rewritten.add(queryRewritten, Occur.MUST);
- rewritten.add(filterRewritten, Occur.FILTER);
- rewritten.setBoost(getBoost());
- return rewritten;
- }
- }
- // nothing to rewrite, we are done!
- return this;
+ BooleanQuery rewritten = new BooleanQuery();
+ rewritten.add(query, Occur.MUST);
+ rewritten.add(strategy.rewrite(filter), Occur.FILTER);
+ rewritten.setBoost(getBoost());
+ return rewritten;
}
/** Returns this FilteredQuery's (unfiltered) Query */
@@ -419,28 +148,22 @@ public class FilteredQuery extends Query
public static final FilterStrategy RANDOM_ACCESS_FILTER_STRATEGY = new RandomAccessFilterStrategy();
/**
- * A filter strategy that uses a "leap-frog" approach (also called "zig-zag join").
- * The scorer and the filter
- * take turns trying to advance to each other's next matching document, often
- * jumping past the target document. When both land on the same document, it's
- * collected.
- * <p>
- * Note: This strategy uses the filter to lead the iteration.
- * </p>
+ * A filter strategy that uses a "leap-frog" approach (also called "zig-zag join").
+ * In spite of the name of this constant, which one will be iterated first depends
+ * on the {@link DocIdSetIterator#cost() cost} of the filter compared to the query.
*/
- public static final FilterStrategy LEAP_FROG_FILTER_FIRST_STRATEGY = new LeapFrogFilterStrategy(false);
+ public static final FilterStrategy LEAP_FROG_FILTER_FIRST_STRATEGY = new RandomAccessFilterStrategy() {
+ protected boolean useRandomAccess(Bits bits, long filterCost) {
+ return false;
+ }
+ };
/**
- * A filter strategy that uses a "leap-frog" approach (also called "zig-zag join").
- * The scorer and the filter
- * take turns trying to advance to each other's next matching document, often
- * jumping past the target document. When both land on the same document, it's
- * collected.
- * <p>
- * Note: This strategy uses the query to lead the iteration.
- * </p>
+ * A filter strategy that uses a "leap-frog" approach (also called "zig-zag join").
+ * In spite of the name of this constant, which one will be iterated first depends
+ * on the {@link DocIdSetIterator#cost() cost} of the filter compared to the query.
*/
- public static final FilterStrategy LEAP_FROG_QUERY_FIRST_STRATEGY = new LeapFrogFilterStrategy(true);
+ public static final FilterStrategy LEAP_FROG_QUERY_FIRST_STRATEGY = LEAP_FROG_FILTER_FIRST_STRATEGY;
/**
* A filter strategy that advances the Query or rather its {@link Scorer} first and consults the
@@ -455,50 +178,21 @@ public class FilteredQuery extends Query
* matching doc like exact geo distances.
* </p>
*/
- public static final FilterStrategy QUERY_FIRST_FILTER_STRATEGY = new QueryFirstFilterStrategy();
+ public static final FilterStrategy QUERY_FIRST_FILTER_STRATEGY = new RandomAccessFilterStrategy() {
+ @Override
+ boolean alwaysUseRandomAccess() {
+ return true;
+ }
+ };
/** Abstract class that defines how the filter ({@link DocIdSet}) applied during document collection. */
public static abstract class FilterStrategy {
-
- /**
- * Returns a filtered {@link Scorer} based on this strategy.
- *
- * @param context
- * the {@link org.apache.lucene.index.LeafReaderContext} for which to return the {@link Scorer}.
- * @param weight the {@link FilteredQuery} {@link Weight} to create the filtered scorer.
- * @param docIdSet the filter {@link DocIdSet} to apply
- * @return a filtered scorer
- *
- * @throws IOException if an {@link IOException} occurs
- */
- public abstract Scorer filteredScorer(LeafReaderContext context,
- Weight weight, DocIdSet docIdSet) throws IOException;
- /**
- * Returns a filtered {@link BulkScorer} based on this
- * strategy. This is an optional method: the default
- * implementation just calls {@link #filteredScorer} and
- * wraps that into a BulkScorer.
- *
- * @param context
- * the {@link org.apache.lucene.index.LeafReaderContext} for which to return the {@link Scorer}.
- * @param weight the {@link FilteredQuery} {@link Weight} to create the filtered scorer.
- * @param docIdSet the filter {@link DocIdSet} to apply
- * @return a filtered top scorer
- */
- public BulkScorer filteredBulkScorer(LeafReaderContext context,
- Weight weight, DocIdSet docIdSet) throws IOException {
- Scorer scorer = filteredScorer(context, weight, docIdSet);
- if (scorer == null) {
- return null;
- }
- // This impl always scores docs in order, so we can
- // ignore scoreDocsInOrder:
- return new Weight.DefaultBulkScorer(scorer);
- }
+ /** Rewrite the filter. */
+ public abstract Query rewrite(Filter filter);
}
-
+
/**
* A {@link FilterStrategy} that conditionally uses a random access filter if
* the given {@link DocIdSet} supports random access (returns a non-null value
@@ -510,27 +204,10 @@ public class FilteredQuery extends Query
public static class RandomAccessFilterStrategy extends FilterStrategy {
@Override
- public Scorer filteredScorer(LeafReaderContext context, Weight weight, DocIdSet docIdSet) throws IOException {
- final DocIdSetIterator filterIter = docIdSet.iterator();
- if (filterIter == null) {
- // this means the filter does not accept any documents.
- return null;
- }
-
- final Bits filterAcceptDocs = docIdSet.bits();
- // force if RA is requested
- final boolean useRandomAccess = filterAcceptDocs != null && useRandomAccess(filterAcceptDocs, filterIter.cost());
- if (useRandomAccess) {
- // if we are using random access, we return the inner scorer, just with other acceptDocs
- return weight.scorer(context, filterAcceptDocs);
- } else {
- // we are gonna advance() this scorer, so we set inorder=true/toplevel=false
- // we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice
- final Scorer scorer = weight.scorer(context, null);
- return (scorer == null) ? null : new LeapFrogScorer(weight, filterIter, scorer, scorer);
- }
+ public Query rewrite(Filter filter) {
+ return new RandomAccessFilterWrapperQuery(filter, this);
}
-
+
/**
* Expert: decides if a filter should be executed as "random-access" or not.
* random-access means the filter "filters" in a similar way as deleted docs are filtered
@@ -547,77 +224,122 @@ public class FilteredQuery extends Query
// if the filter matches more than 1% of documents, we use random-access
return filterCost * 100 > bits.length();
}
+
+ // back door for QUERY_FIRST_FILTER_STRATEGY, when this returns true we
+ // will try to use the random-access API regardless of the iterator
+ boolean alwaysUseRandomAccess() {
+ return false;
+ }
}
-
- private static final class LeapFrogFilterStrategy extends FilterStrategy {
-
- private final boolean scorerFirst;
-
- private LeapFrogFilterStrategy(boolean scorerFirst) {
- this.scorerFirst = scorerFirst;
+
+ private static class RandomAccessFilterWrapperQuery extends Query {
+
+ final Filter filter;
+ final RandomAccessFilterStrategy strategy;
+
+ private RandomAccessFilterWrapperQuery(Filter filter, RandomAccessFilterStrategy strategy) {
+ this.filter = Objects.requireNonNull(filter);
+ this.strategy = Objects.requireNonNull(strategy);
}
@Override
- public Scorer filteredScorer(LeafReaderContext context,
- Weight weight, DocIdSet docIdSet) throws IOException {
- final DocIdSetIterator filterIter = docIdSet.iterator();
- if (filterIter == null) {
- // this means the filter does not accept any documents.
- return null;
- }
- // we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice
- final Scorer scorer = weight.scorer(context, null);
- if (scorer == null) {
- return null;
+ public boolean equals(Object obj) {
+ if (super.equals(obj) == false) {
+ return false;
}
+ RandomAccessFilterWrapperQuery that = (RandomAccessFilterWrapperQuery) obj;
+ return filter.equals(that.filter) && strategy.equals(that.strategy);
+ }
- if (scorerFirst) {
- return new LeapFrogScorer(weight, scorer, filterIter, scorer);
- } else {
- return new LeapFrogScorer(weight, filterIter, scorer, scorer);
- }
+ @Override
+ public int hashCode() {
+ return 31 * super.hashCode() + Objects.hash(filter, strategy);
}
- }
-
- /**
- * A filter strategy that advances the {@link Scorer} first and consults the
- * {@link DocIdSet} for each matched document.
- * <p>
- * Note: this strategy requires a {@link DocIdSet#bits()} to return a non-null value. Otherwise
- * this strategy falls back to {@link FilteredQuery#LEAP_FROG_QUERY_FIRST_STRATEGY}
- * </p>
- * <p>
- * Use this strategy if the filter computation is more expensive than document
- * scoring or if the filter has a linear running time to compute the next
- * matching doc like exact geo distances.
- * </p>
- */
- private static final class QueryFirstFilterStrategy extends FilterStrategy {
+
@Override
- public Scorer filteredScorer(final LeafReaderContext context,
- Weight weight, DocIdSet docIdSet) throws IOException {
- Bits filterAcceptDocs = docIdSet.bits();
- if (filterAcceptDocs == null) {
- // Filter does not provide random-access Bits; we
- // must fallback to leapfrog:
- return LEAP_FROG_QUERY_FIRST_STRATEGY.filteredScorer(context, weight, docIdSet);
- }
- final Scorer scorer = weight.scorer(context, null);
- return scorer == null ? null : new QueryFirstScorer(weight, filterAcceptDocs, scorer);
+ public String toString(String field) {
+ return filter.toString(field);
}
@Override
- public BulkScorer filteredBulkScorer(final LeafReaderContext context,
- Weight weight, DocIdSet docIdSet) throws IOException {
- Bits filterAcceptDocs = docIdSet.bits();
- if (filterAcceptDocs == null) {
- // Filter does not provide random-access Bits; we
- // must fallback to leapfrog:
- return LEAP_FROG_QUERY_FIRST_STRATEGY.filteredBulkScorer(context, weight, docIdSet);
- }
- final Scorer scorer = weight.scorer(context, null);
- return scorer == null ? null : new QueryFirstBulkScorer(scorer, filterAcceptDocs);
+ public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+ return new Weight(this) {
+
+ @Override
+ public void extractTerms(Set<Term> terms) {}
+
+ @Override
+ public float getValueForNormalization() throws IOException {
+ return 0f;
+ }
+
+ @Override
+ public void normalize(float norm, float topLevelBoost) {}
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ final Scorer s = scorer(context, context.reader().getLiveDocs());
+ final boolean match;
+ if (s == null) {
+ match = false;
+ } else {
+ final TwoPhaseIterator twoPhase = s.asTwoPhaseIterator();
+ if (twoPhase == null) {
+ match = s.advance(doc) == doc;
+ } else {
+ match = twoPhase.approximation().advance(doc) == doc && twoPhase.matches();
+ }
+ }
+ if (match) {
+ assert s.score() == 0f;
+ return Explanation.match(0f, "Match on id " + doc);
+ } else {
+ return Explanation.match(0f, "No match on id " + doc);
+ }
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
+ final DocIdSet set = filter.getDocIdSet(context, acceptDocs);
+ if (set == null) {
+ return null;
+ }
+ final Bits bits = set.bits();
+ boolean useRandomAccess = bits != null && strategy.alwaysUseRandomAccess();
+ final DocIdSetIterator iterator;
+ if (useRandomAccess) {
+ // we don't need the iterator
+ iterator = null;
+ } else {
+ iterator = set.iterator();
+ if (iterator == null) {
+ return null;
+ }
+ if (bits != null) {
+ useRandomAccess = strategy.useRandomAccess(bits, iterator.cost());
+ }
+ }
+
+ if (useRandomAccess) {
+ // use the random-access API
+ final DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
+ final TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
+ @Override
+ public boolean matches() throws IOException {
+ final int doc = approximation.docID();
+ return bits.get(doc);
+ }
+ };
+ return new ConstantScoreScorer(this, 0f, twoPhase);
+ } else {
+ // use the iterator API
+ return new ConstantScoreScorer(this, 0f, iterator);
+ }
+ }
+
+ };
}
+
}
-
+
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java?rev=1687514&r1=1687513&r2=1687514&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestCachingWrapperQuery.java Thu Jun 25 13:19:10 2015
@@ -139,15 +139,15 @@ public class TestCachingWrapperQuery ext
CachingWrapperQuery cacher = new CachingWrapperQuery(filter, QueryCachingPolicy.ALWAYS_CACHE);
// first time, nested filter is called
- cacher.createWeight(searcher, false).scorer(context, context.reader().getLiveDocs());
+ searcher.rewrite(cacher).createWeight(searcher, false).scorer(context, context.reader().getLiveDocs());
assertTrue("first time", filter.wasCalled());
// make sure no exception if cache is holding the wrong docIdSet
- cacher.createWeight(searcher, false).scorer(context, context.reader().getLiveDocs());
+ searcher.rewrite(cacher).createWeight(searcher, false).scorer(context, context.reader().getLiveDocs());
// second time, nested filter should not be called
filter.clear();
- cacher.createWeight(searcher, false).scorer(context, context.reader().getLiveDocs());
+ searcher.rewrite(cacher).createWeight(searcher, false).scorer(context, context.reader().getLiveDocs());
assertFalse("second time", filter.wasCalled());
reader.close();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java?rev=1687514&r1=1687513&r2=1687514&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java Thu Jun 25 13:19:10 2015
@@ -320,7 +320,14 @@ public class TestFilteredQuery extends L
public String toString(String field) {
return in.toString(field);
}
-
+
+ @Override
+ public boolean equals(Object obj) {
+ if (super.equals(obj) == false) {
+ return false;
+ }
+ return in.equals(((FilterWrapper) obj).in);
+ }
}
private void tChainedFilters(final boolean useRandomAccess) throws Exception {
@@ -384,37 +391,6 @@ public class TestFilteredQuery extends L
return randomFilterStrategy(random(), true);
}
- private void assertRewrite(FilteredQuery fq, Class<? extends Query> clazz) throws Exception {
- // assign crazy boost to FQ
- final float boost = random().nextFloat() * 100.f;
- fq.setBoost(boost);
-
-
- // assign crazy boost to inner
- final float innerBoost = random().nextFloat() * 100.f;
- fq.getQuery().setBoost(innerBoost);
-
- // check the class and boosts of rewritten query
- final Query rewritten = searcher.rewrite(fq);
- assertTrue("is not instance of " + clazz.getName(), clazz.isInstance(rewritten));
- if (rewritten instanceof FilteredQuery) {
- assertEquals(boost, rewritten.getBoost(), 1.E-5f);
- assertEquals(innerBoost, ((FilteredQuery) rewritten).getQuery().getBoost(), 1.E-5f);
- assertEquals(fq.getFilterStrategy(), ((FilteredQuery) rewritten).getFilterStrategy());
- } else {
- assertEquals(boost * innerBoost, rewritten.getBoost(), 1.E-5f);
- }
-
- // check that the original query was not modified
- assertEquals(boost, fq.getBoost(), 1.E-5f);
- assertEquals(innerBoost, fq.getQuery().getBoost(), 1.E-5f);
- }
-
- public void testRewrite() throws Exception {
- assertRewrite(new FilteredQuery(new TermQuery(new Term("field", "one")), new FilterWrapper(new QueryWrapperFilter(new PrefixQuery(new Term("field", "o")))), randomFilterStrategy()), FilteredQuery.class);
- assertRewrite(new FilteredQuery(new PrefixQuery(new Term("field", "one")), new FilterWrapper(new QueryWrapperFilter(new PrefixQuery(new Term("field", "o")))), randomFilterStrategy()), FilteredQuery.class);
- }
-
public void testGetFilterStrategy() {
FilterStrategy randomFilterStrategy = randomFilterStrategy();
FilteredQuery filteredQuery = new FilteredQuery(new TermQuery(new Term("field", "one")), new QueryWrapperFilter(new PrefixQuery(new Term("field", "o"))), randomFilterStrategy);
@@ -456,6 +432,7 @@ public class TestFilteredQuery extends L
writer.close();
IndexSearcher searcher = newSearcher(reader);
+ searcher.setQueryCache(null); // needed otherwise the iterator may be used
Query query = new FilteredQuery(new TermQuery(new Term("field", "0")),
new Filter() {
@Override
@@ -521,95 +498,6 @@ public class TestFilteredQuery extends L
assertEquals(totalDocsWithZero, search.totalHits);
IOUtils.close(reader, directory);
}
-
- /*
- * Test if the leapfrog strategy works correctly in terms
- * of advancing / next the right thing first
- */
- public void testLeapFrogStrategy() throws IOException {
- Directory directory = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter (random(), directory, newIndexWriterConfig(new MockAnalyzer(random())));
- int numDocs = atLeast(50);
- int totalDocsWithZero = 0;
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- int num = random().nextInt(10);
- if (num == 0) {
- totalDocsWithZero++;
- }
- doc.add (newTextField("field", ""+num, Field.Store.YES));
- writer.addDocument (doc);
- }
- IndexReader reader = writer.getReader();
- writer.close();
- final boolean queryFirst = random().nextBoolean();
- IndexSearcher searcher = newSearcher(reader);
- Query query = new FilteredQuery(new TermQuery(new Term("field", "0")), new Filter() {
- @Override
- public DocIdSet getDocIdSet(final LeafReaderContext context, Bits acceptDocs)
- throws IOException {
- return new DocIdSet() {
-
- @Override
- public long ramBytesUsed() {
- return 0L;
- }
-
- @Override
- public Bits bits() throws IOException {
- return null;
- }
- @Override
- public DocIdSetIterator iterator() throws IOException {
- final PostingsEnum termPostingsEnum = context.reader().postings(new Term("field", "0"));
- if (termPostingsEnum == null) {
- return null;
- }
- return new DocIdSetIterator() {
- boolean nextCalled;
- boolean advanceCalled;
- @Override
- public int nextDoc() throws IOException {
- assertTrue("queryFirst: "+ queryFirst + " advanced: " + advanceCalled + " next: "+ nextCalled, nextCalled || advanceCalled ^ !queryFirst);
- nextCalled = true;
- return termPostingsEnum.nextDoc();
- }
-
- @Override
- public int docID() {
- return termPostingsEnum.docID();
- }
-
- @Override
- public int advance(int target) throws IOException {
- assertTrue("queryFirst: "+ queryFirst + " advanced: " + advanceCalled + " next: "+ nextCalled, advanceCalled || nextCalled ^ queryFirst);
- advanceCalled = true;
- return termPostingsEnum.advance(target);
- }
-
- @Override
- public long cost() {
- return termPostingsEnum.cost();
- }
- };
- }
-
-
- };
-
- }
- @Override
- public String toString(String field) {
- return "filterField0";
- }
- }, queryFirst ? FilteredQuery.LEAP_FROG_QUERY_FIRST_STRATEGY : random()
- .nextBoolean() ? FilteredQuery.RANDOM_ACCESS_FILTER_STRATEGY
- : FilteredQuery.LEAP_FROG_FILTER_FIRST_STRATEGY); // if filterFirst, we can use random here since bits are null
-
- TopDocs search = searcher.search(query, 10);
- assertEquals(totalDocsWithZero, search.totalHits);
- IOUtils.close(reader, directory);
- }
public void testPreservesScores() throws IOException {
Directory dir = newDirectory();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java?rev=1687514&r1=1687513&r2=1687514&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestScorerPerf.java Thu Jun 25 13:19:10 2015
@@ -45,15 +45,18 @@ public class TestScorerPerf extends Luce
Directory d;
// TODO: this should be setUp()....
- public void createDummySearcher() throws Exception {
+ public void createDummySearcher(int maxDoc) throws Exception {
// Create a dummy index with nothing in it.
// This could possibly fail if Lucene starts checking for docid ranges...
d = newDirectory();
IndexWriter iw = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));
- iw.addDocument(new Document());
- iw.close();
- r = DirectoryReader.open(d);
+ for (int i = 0; i < maxDoc; ++i) {
+ iw.addDocument(new Document());
+ }
+ iw.forceMerge(1);
+ r = DirectoryReader.open(iw, false);
s = newSearcher(r);
+ iw.close();
}
public void createRandomTerms(int nDocs, int nTerms, double power, Directory dir) throws Exception {
@@ -150,6 +153,7 @@ public class TestScorerPerf extends Luce
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
assertNull("acceptDocs should be null, as we have an index without deletions", acceptDocs);
+ assertEquals(context.reader().maxDoc(), docs.length());
return new BitDocIdSet(docs);
}
@@ -174,7 +178,7 @@ public class TestScorerPerf extends Luce
FixedBitSet addClause(BooleanQuery bq, FixedBitSet result) {
final FixedBitSet rnd = sets[random().nextInt(sets.length)];
- Query q = new ConstantScoreQuery(new BitSetFilter(rnd));
+ Query q = new BitSetFilter(rnd);
bq.add(q, BooleanClause.Occur.MUST);
if (validate) {
if (result==null) result = rnd.clone();
@@ -342,9 +346,10 @@ public class TestScorerPerf extends Luce
public void testConjunctions() throws Exception {
// test many small sets... the bugs will be found on boundary conditions
- createDummySearcher();
validate=true;
- sets=randBitSets(atLeast(1000), atLeast(10));
+ final int maxDoc = atLeast(10);
+ createDummySearcher(maxDoc);
+ sets=randBitSets(atLeast(1000), maxDoc);
doConjunctions(atLeast(10000), atLeast(5));
doNestedConjunctions(atLeast(10000), atLeast(3), atLeast(3));
r.close();
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java?rev=1687514&r1=1687513&r2=1687514&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java Thu Jun 25 13:19:10 2015
@@ -181,7 +181,7 @@ public abstract class SearchEquivalenceT
if (q != query) {
return new SlowWrapperFilter(q, useBits);
} else {
- return this;
+ return super.rewrite(reader);
}
}