You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2026/06/30 10:21:58 UTC
(lucene) branch branch_10x updated: Optimize cached-filter conjunction batching (#16146)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 2f59ef1bd6e Optimize cached-filter conjunction batching (#16146)
2f59ef1bd6e is described below
commit 2f59ef1bd6eb210a832bba94a02ce8cc203f7e18
Author: Costin Leau <co...@users.noreply.github.com>
AuthorDate: Tue Jun 30 02:30:12 2026 -0700
Optimize cached-filter conjunction batching (#16146)
Speed up no-score filtered-optional Boolean queries by routing dense
conjunctions with cached FixedBitSet filters through ConstantScoreBulkScorer,
which batches matches in bulk via DocIdStream.
---
lucene/CHANGES.txt | 3 +
.../jmh/CachedFilterConjunctionBenchmark.java | 375 ++++++++++++++++++++
.../lucene/search/BooleanScorerSupplier.java | 10 +-
.../org/apache/lucene/search/ConjunctionDISI.java | 46 +++
.../apache/lucene/search/ConstantScoreQuery.java | 10 +
.../apache/lucene/search/FilterLeafCollector.java | 6 +
.../apache/lucene/search/TestBooleanScorer.java | 382 ++++++++++++++++++++-
.../apache/lucene/search/TestConjunctionDISI.java | 173 ++++++++++
.../lucene/search/TestConstantScoreQuery.java | 116 +++++++
9 files changed, 1112 insertions(+), 9 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 821d34f79d4..76b205179c0 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -21,6 +21,9 @@ Optimizations
---------------------
* GITHUG#16280: Single-pass writeString fast path for short strings in ByteBuffersDataOutput (neoremind)
+* GITHUB#16146: Speed up no-score filtered-optional Boolean queries by routing dense conjunctions with cached FixedBitSet filters
+ through ConstantScoreBulkScorer, which batches matches in bulk via DocIdStream. (Costin Leau)
+
Bug Fixes
---------------------
(No changes)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CachedFilterConjunctionBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CachedFilterConjunctionBenchmark.java
new file mode 100644
index 00000000000..23591865a3e
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/CachedFilterConjunctionBenchmark.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BulkScorer;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.DocIdStream;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LRUQueryCache;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryCachingPolicy;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * Benchmarks cached-filter conjunctions on no-score Boolean queries.
+ *
+ * <p>The filter is warmed into {@link LRUQueryCache} before measurement. Dense filter selectivities
+ * are above the cache's FixedBitSet threshold, while very sparse selectivities exercise the
+ * RoaringDocIdSet cache path.
+ *
+ * <p>The {@code filtered_optional} shape exercises the production path where a cached filter is
+ * combined with a sparse optional disjunction through {@code
+ * BooleanScorerSupplier.filteredOptionalBulkScorer()}, which can route to {@code
+ * ConstantScoreBulkScorer}. The {@code filter_only} shape is retained as a control for the older
+ * pure-filter conjunction shape. Very sparse filters below the query-cache bitset threshold and
+ * very dense optional leads are controls too: they exercise RoaringDocIdSet and
+ * DenseConjunctionBulkScorer paths rather than {@code BitSetConjunctionDISI.intoBitSet()}.
+ */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 3, time = 3)
+@Measurement(iterations = 5, time = 5)
+@Fork(
+ value = 1,
+ warmups = 1,
+ jvmArgsAppend = {"-Xmx2g", "-Xms2g", "-XX:+AlwaysPreTouch"})
+public class CachedFilterConjunctionBenchmark {
+
+ private static final String FILTER_FIELD = "filter";
+ private static final String LEAD_FIELD = "lead";
+ private static final String OPTIONAL_FIELD = "optional";
+ private static final String OPTIONAL_A = "a";
+ private static final String OPTIONAL_B = "b";
+ private static final String YES = "yes";
+ private static final String BASELINE = "baseline";
+ private static final String BULKSCORER = "bulkscorer";
+ private static final String FILTER_ONLY = "filter_only";
+ private static final String FILTERED_OPTIONAL = "filtered_optional";
+
+ private Directory dir;
+ private IndexReader reader;
+ private Path path;
+ private IndexSearcher cachedSearcher;
+ private IndexSearcher uncachedSearcher;
+ private LRUQueryCache queryCache;
+ private Query filterQuery;
+ private Query conjunctionQuery;
+ private Query filterOnlyQuery;
+ private Query filteredOptionalQuery;
+ private LeafReaderContext context;
+ private Weight cachedWeight;
+ private Weight uncachedWeight;
+ private boolean cachedFilterUsesBitSet;
+ private int expectedHitCount;
+
+ @State(Scope.Benchmark)
+ public static class Params {
+ @Param({"1000000"})
+ public int docCount;
+
+ @Param({"0.0001", "0.001", "0.002", "0.003", "0.005", "0.01", "0.03", "0.10", "0.50", "1.0"})
+ public double filterSelectivity;
+
+ @Param({"0.001", "0.03", "0.10"})
+ public double leadSelectivity;
+
+ @Param({FILTERED_OPTIONAL, FILTER_ONLY})
+ public String queryShape;
+
+ @Param({BASELINE, BULKSCORER})
+ public String variant;
+ }
+
+ @Setup(Level.Trial)
+ public void setup(Params params) throws Exception {
+ path = Files.createTempDirectory("cachedFilterConjunctionBench");
+ dir = MMapDirectory.open(path);
+
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig());
+ Random random = new Random(42);
+ for (int i = 0; i < params.docCount; ++i) {
+ Document doc = new Document();
+ if (random.nextDouble() < params.filterSelectivity) {
+ doc.add(new StringField(FILTER_FIELD, YES, Store.NO));
+ }
+ if (random.nextDouble() < params.leadSelectivity) {
+ doc.add(new StringField(LEAD_FIELD, YES, Store.NO));
+ if (random.nextBoolean()) {
+ doc.add(new StringField(OPTIONAL_FIELD, OPTIONAL_A, Store.NO));
+ } else {
+ doc.add(new StringField(OPTIONAL_FIELD, OPTIONAL_B, Store.NO));
+ }
+ }
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ reader = DirectoryReader.open(w);
+ w.close();
+
+ filterQuery = new TermQuery(new Term(FILTER_FIELD, YES));
+ Query leadQuery = new TermQuery(new Term(LEAD_FIELD, YES));
+ filterOnlyQuery =
+ new BooleanQuery.Builder()
+ .add(leadQuery, Occur.FILTER)
+ .add(filterQuery, Occur.FILTER)
+ .build();
+ filteredOptionalQuery =
+ new BooleanQuery.Builder()
+ .add(filterQuery, Occur.FILTER)
+ .add(new TermQuery(new Term(OPTIONAL_FIELD, OPTIONAL_A)), Occur.SHOULD)
+ .add(new TermQuery(new Term(OPTIONAL_FIELD, OPTIONAL_B)), Occur.SHOULD)
+ .setMinimumNumberShouldMatch(1)
+ .build();
+ conjunctionQuery =
+ switch (params.queryShape) {
+ case FILTER_ONLY -> filterOnlyQuery;
+ case FILTERED_OPTIONAL -> filteredOptionalQuery;
+ default -> throw new AssertionError("Unknown query shape: " + params.queryShape);
+ };
+
+ queryCache = new LRUQueryCache(256, 32L * 1024 * 1024, context -> context.reader() != null, 1f);
+ cachedSearcher = new IndexSearcher(reader);
+ cachedSearcher.setQueryCache(queryCache);
+ cachedSearcher.setQueryCachingPolicy(cacheOnly(filterQuery));
+
+ uncachedSearcher = new IndexSearcher(reader);
+ uncachedSearcher.setQueryCache(null);
+
+ context = reader.leaves().get(0);
+
+ // Warm only the filter query so the benchmark measures reuse of a cached FixedBitSet filter,
+ // not first-use cache population or a fully cached top-level conjunction.
+ Weight filterWeight =
+ cachedSearcher.createWeight(
+ cachedSearcher.rewrite(filterQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
+ ScorerSupplier filterScorerSupplier = filterWeight.scorerSupplier(context);
+ if (filterScorerSupplier != null) {
+ cachedFilterUsesBitSet = filterScorerSupplier.cost() * 100 >= context.reader().maxDoc();
+ filterScorerSupplier.get(Long.MAX_VALUE).iterator().nextDoc();
+ }
+ if (queryCache.getCacheSize() == 0) {
+ throw new AssertionError("filter query was not cached");
+ }
+
+ cachedWeight =
+ cachedSearcher.createWeight(
+ cachedSearcher.rewrite(conjunctionQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
+ uncachedWeight =
+ uncachedSearcher.createWeight(
+ uncachedSearcher.rewrite(conjunctionQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
+ expectedHitCount = uncachedSearcher.count(conjunctionQuery);
+ checkBulkScorerRoute(params, cachedWeight);
+ checkHitCount(scoreVariant(cachedWeight, params.variant));
+ checkHitCount(scoreVariant(uncachedWeight, params.variant));
+ }
+
+ private static QueryCachingPolicy cacheOnly(Query queryToCache) {
+ return new QueryCachingPolicy() {
+ @Override
+ public void onUse(Query query) {}
+
+ @Override
+ public boolean shouldCache(Query query) {
+ return queryToCache.equals(query);
+ }
+ };
+ }
+
+ @Benchmark
+ public int countCachedFilterConjunction(Params params) throws IOException {
+ return scoreVariant(cachedWeight, params.variant);
+ }
+
+ @Benchmark
+ public int countUncachedFilterConjunction(Params params) throws IOException {
+ return scoreVariant(uncachedWeight, params.variant);
+ }
+
+ private int scoreVariant(Weight weight, String variant) throws IOException {
+ switch (variant) {
+ case BASELINE:
+ Scorer scorer = scorer(weight);
+ if (scorer == null) {
+ return 0;
+ }
+ return scorePerDoc(scorer);
+ case BULKSCORER:
+ return scoreBulkScorer(weight);
+ default:
+ throw new AssertionError("Unknown variant: " + variant);
+ }
+ }
+
+ private Scorer scorer(Weight weight) throws IOException {
+ ScorerSupplier scorerSupplier = weight.scorerSupplier(context);
+ if (scorerSupplier == null) {
+ return null;
+ }
+ return scorerSupplier.get(Long.MAX_VALUE);
+ }
+
+ private int scorePerDoc(Scorer scorer) throws IOException {
+ DocIdSetIterator iterator = scorer.iterator();
+ int maxDoc = context.reader().maxDoc();
+ return scorePerDoc(iterator, maxDoc);
+ }
+
+ private int scorePerDoc(DocIdSetIterator iterator, int maxDoc) throws IOException {
+ CountingLeafCollector collector = new CountingLeafCollector();
+ for (int doc = iterator.nextDoc(); doc < maxDoc; doc = iterator.nextDoc()) {
+ collector.collect(doc);
+ }
+ return collector.count;
+ }
+
+ private int scoreBulkScorer(Weight weight) throws IOException {
+ BulkScorer bulkScorer = weight.bulkScorer(context);
+ if (bulkScorer == null) {
+ return 0;
+ }
+ CountingLeafCollector collector = new CountingLeafCollector();
+ bulkScorer.score(collector, context.reader().getLiveDocs(), 0, context.reader().maxDoc());
+ return collector.count;
+ }
+
+ private void checkBulkScorerRoute(Params params, Weight weight) throws IOException {
+ ScorerSupplier scorerSupplier = weight.scorerSupplier(context);
+ if (scorerSupplier == null) {
+ return;
+ }
+ BulkScorer bulkScorer = scorerSupplier.bulkScorer();
+ String bulkScorerClassName = bulkScorer.getClass().getName();
+ boolean isConstantScoreBulkScorer = bulkScorerClassName.endsWith(".ConstantScoreBulkScorer");
+ boolean isDenseConjunctionBulkScorer =
+ bulkScorerClassName.endsWith(".DenseConjunctionBulkScorer");
+ boolean isDefaultBulkScorer = bulkScorerClassName.endsWith("$DefaultBulkScorer");
+ if (FILTERED_OPTIONAL.equals(params.queryShape)
+ && isConstantScoreBulkScorer == false
+ && isDenseConjunctionBulkScorer == false
+ && isDefaultBulkScorer == false) {
+ throw new AssertionError(
+ "filtered_optional should route through ConstantScoreBulkScorer, "
+ + "DenseConjunctionBulkScorer, or DefaultBulkScorer but got "
+ + bulkScorerClassName);
+ }
+ if (FILTERED_OPTIONAL.equals(params.queryShape)
+ && isConstantScoreBulkScorer
+ && cachedFilterUsesBitSet
+ && params.filterSelectivity >= 0.02) {
+ Scorer scorer = scorerSupplier.get(Long.MAX_VALUE);
+ String iteratorClassName = scorer.iterator().getClass().getName();
+ if (iteratorClassName.endsWith("$BitSetConjunctionDISI") == false) {
+ throw new AssertionError(
+ "cached filter should produce BitSetConjunctionDISI but got " + iteratorClassName);
+ }
+ }
+ if (FILTER_ONLY.equals(params.queryShape) && isConstantScoreBulkScorer) {
+ throw new AssertionError("filter_only unexpectedly routed through ConstantScoreBulkScorer");
+ }
+ }
+
+ private void checkHitCount(int hitCount) {
+ if (hitCount != expectedHitCount) {
+ throw new AssertionError("hitCount=" + hitCount + " expected=" + expectedHitCount);
+ }
+ }
+
+ private static class CountingLeafCollector implements LeafCollector {
+ int count;
+
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ ++count;
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ // Match TotalHitCountCollector's batch path.
+ for (int streamCount = stream.count(); streamCount != 0; streamCount = stream.count()) {
+ count += streamCount;
+ }
+ }
+ }
+
+ @TearDown(Level.Trial)
+ public void tearDown() throws IOException {
+ if (reader != null) {
+ reader.close();
+ }
+ if (dir != null) {
+ dir.close();
+ }
+ if (path != null && Files.exists(path)) {
+ try (Stream<Path> walk = Files.walk(path)) {
+ walk.sorted(Comparator.reverseOrder())
+ .forEach(
+ p -> {
+ try {
+ Files.delete(p);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ });
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java
index 01647a6ba31..85df76b0b45 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java
@@ -353,7 +353,15 @@ final class BooleanScorerSupplier extends ScorerSupplier {
return DenseConjunctionBulkScorer.of(filters, maxDoc, 0f);
}
- return new DefaultBulkScorer(new ConjunctionScorer(filters, Collections.emptyList()));
+ Scorer scorer = new ConjunctionScorer(filters, Collections.emptyList());
+ DocIdSetIterator iterator = scorer.iterator();
+ if ((cost >> 1) >= DenseConjunctionBulkScorer.WINDOW_SIZE) {
+ TwoPhaseIterator twoPhase = TwoPhaseIterator.unwrap(iterator);
+ return twoPhase == null
+ ? new ConstantScoreBulkScorer(0f, scoreMode, iterator)
+ : new ConstantScoreBulkScorer(0f, scoreMode, twoPhase);
+ }
+ return new DefaultBulkScorer(scorer);
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/ConjunctionDISI.java b/lucene/core/src/java/org/apache/lucene/search/ConjunctionDISI.java
index 9cf17c1b5ad..126cd4d0b03 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ConjunctionDISI.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ConjunctionDISI.java
@@ -25,6 +25,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.CollectionUtil;
+import org.apache.lucene.util.FixedBitSet;
/**
* A conjunction of DocIdSetIterators. Requires that all of its sub-iterators must be on the same
@@ -229,6 +230,7 @@ final class ConjunctionDISI extends FilterDocIdSetIterator {
private final BitSetIterator[] bitSetIterators;
private final BitSet[] bitSets;
private final int minLength;
+ private FixedBitSet scratch;
BitSetConjunctionDISI(DocIdSetIterator lead, Collection<BitSetIterator> bitSetIterators) {
super(lead);
@@ -262,6 +264,47 @@ final class ConjunctionDISI extends FilterDocIdSetIterator {
return doNext(lead.advance(target));
}
+ @Override
+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
+ assert offset <= docID() : "offset=" + offset + " docID()=" + docID() + " upTo=" + upTo;
+ int doc = docID();
+ if (doc >= upTo) {
+ return;
+ }
+
+ // Bulk masking has fixed per-window cost; sparse leads are cheaper to advance per doc.
+ if (lead.cost() < bitSet.length()) {
+ super.intoBitSet(upTo, bitSet, offset);
+ return;
+ }
+
+ int bulkUpTo = Math.min(upTo, minLength);
+ long destinationEnd = (long) offset + bitSet.length();
+ if (destinationEnd < bulkUpTo) {
+ bulkUpTo = (int) destinationEnd;
+ }
+
+ if (doc < bulkUpTo) {
+ if (scratch == null || scratch.length() != bitSet.length()) {
+ scratch = new FixedBitSet(bitSet.length());
+ } else {
+ scratch.clear();
+ }
+
+ lead.intoBitSet(bulkUpTo, scratch, offset);
+ for (BitSet bitSetMask : bitSets) {
+ bitSetMask.applyMask(scratch, offset);
+ }
+ bitSet.or(scratch);
+
+ doNext(lead.docID());
+ }
+
+ if (docID() < upTo) {
+ super.intoBitSet(upTo, bitSet, offset);
+ }
+ }
+
private int doNext(int doc) throws IOException {
advanceLead:
for (; ; doc = lead.nextDoc()) {
@@ -269,6 +312,9 @@ final class ConjunctionDISI extends FilterDocIdSetIterator {
if (doc != NO_MORE_DOCS) {
lead.advance(NO_MORE_DOCS);
}
+ for (BitSetIterator iterator : bitSetIterators) {
+ iterator.setDocId(NO_MORE_DOCS);
+ }
return NO_MORE_DOCS;
}
for (BitSet bitSet : bitSets) {
diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
index bfc06c0986b..3e66a1e6803 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
@@ -112,6 +112,16 @@ public final class ConstantScoreQuery extends Query {
}
});
}
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ in.collect(stream);
+ }
+
+ @Override
+ public void collectRange(int min, int max) throws IOException {
+ in.collectRange(min, max);
+ }
};
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java b/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java
index d9bc671fd42..173ef24676a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FilterLeafCollector.java
@@ -21,6 +21,12 @@ import java.io.IOException;
/**
* {@link LeafCollector} delegator.
*
+ * <p>This collector inherits the default batch collection behavior from {@link LeafCollector},
+ * which decomposes batches through {@link #collect(int)}. This preserves per-document behavior for
+ * subclasses that override {@link #collect(int)}. Subclasses that only delegate and want to
+ * preserve bulk-collection optimizations may override {@link #collect(DocIdStream)} and {@link
+ * #collectRange(int, int)} to forward directly to the wrapped collector.
+ *
* @lucene.experimental
*/
public abstract class FilterLeafCollector implements LeafCollector {
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
index 399a73ffaa8..571c7ff7c23 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
@@ -27,8 +27,12 @@ import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -38,6 +42,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.search.QueryUtils;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
@@ -253,6 +258,370 @@ public class TestBooleanScorer extends LuceneTestCase {
assertEquals(3, iterator.intoBitSetCalls);
}
+ public void testCachedFilterOptionalConjunctionUsesConstantScoreBulkScorerDocIdStream()
+ throws IOException {
+ int maxDoc = 500_000;
+ int expectedHitCount = 10_000;
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig());
+ for (int doc = 0; doc < maxDoc; ++doc) {
+ Document document = new Document();
+ document.add(new StringField("filter", "yes", Store.NO));
+ if (doc % 50 == 1) {
+ document.add(new StringField("optional", doc % 100 == 1 ? "a" : "b", Store.NO));
+ }
+ w.addDocument(document);
+ }
+ w.forceMerge(1);
+ IndexReader reader = DirectoryReader.open(w);
+ w.close();
+
+ Query filterQuery = new TermQuery(new Term("filter", "yes"));
+ Query query =
+ new BooleanQuery.Builder()
+ .add(filterQuery, Occur.FILTER)
+ .add(new TermQuery(new Term("optional", "a")), Occur.SHOULD)
+ .add(new TermQuery(new Term("optional", "b")), Occur.SHOULD)
+ .setMinimumNumberShouldMatch(1)
+ .build();
+
+ IndexSearcher searcher = new IndexSearcher(reader);
+ LRUQueryCache queryCache =
+ new LRUQueryCache(16, 1 << 25, context -> context.reader() != null, 1f);
+ searcher.setQueryCache(queryCache);
+ searcher.setQueryCachingPolicy(
+ new QueryCachingPolicy() {
+ @Override
+ public void onUse(Query query) {}
+
+ @Override
+ public boolean shouldCache(Query query) {
+ return filterQuery.equals(query);
+ }
+ });
+
+ LeafReaderContext context = reader.leaves().get(0);
+ Weight filterWeight =
+ searcher.createWeight(searcher.rewrite(filterQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
+ ScorerSupplier filterScorerSupplier = filterWeight.scorerSupplier(context);
+ assertNotNull(filterScorerSupplier);
+ filterScorerSupplier.get(Long.MAX_VALUE).iterator().nextDoc();
+ assertEquals(1, queryCache.getCacheSize());
+
+ Weight weight =
+ searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f);
+ Scorer scorer = weight.scorerSupplier(context).get(Long.MAX_VALUE);
+ assertThat(scorer, instanceOf(ConjunctionScorer.class));
+ DocIdSetIterator iterator = scorer.iterator();
+ assertTrue(iterator.cost() >= Math.ceilDiv(maxDoc, 512));
+ assertNull(TwoPhaseIterator.unwrap(iterator));
+
+ BulkScorer bulkScorer = weight.bulkScorer(context);
+ assertThat(bulkScorer, instanceOf(ConstantScoreBulkScorer.class));
+
+ int[] collected = new int[expectedHitCount];
+ int[] count = new int[1];
+ int[] streamCount = new int[1];
+ LeafCollector collector =
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ fail("Cached filtered optional conjunction should collect via ConstantScoreBulkScorer");
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ streamCount[0]++;
+ int[] buffer = new int[32];
+ for (int size = stream.intoArray(buffer); size != 0; size = stream.intoArray(buffer)) {
+ System.arraycopy(buffer, 0, collected, count[0], size);
+ count[0] += size;
+ }
+ }
+ };
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, bulkScorer.score(collector, null, 0, maxDoc));
+ assertTrue(streamCount[0] > 0);
+ assertEquals(expectedHitCount, count[0]);
+ for (int i = 0; i < count[0]; ++i) {
+ assertEquals(1 + i * 50, collected[i]);
+ }
+
+ reader.close();
+ dir.close();
+ }
+
+ public void testCachedFilterTwoPhaseOptionalConjunctionUsesConstantScoreBulkScorer()
+ throws IOException {
+ int maxDoc = 500_000;
+ int expectedHitCount = 10_000;
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig());
+ for (int doc = 0; doc < maxDoc; ++doc) {
+ Document document = new Document();
+ document.add(new StringField("filter", "yes", Store.NO));
+ if (doc % 50 == 1) {
+ // Phrase queries expose a TwoPhaseIterator, so the optional disjunction below is two-phase.
+ document.add(
+ new TextField("optional", doc % 100 == 1 ? "quick brown" : "lazy dog", Store.NO));
+ }
+ w.addDocument(document);
+ }
+ w.forceMerge(1);
+ IndexReader reader = DirectoryReader.open(w);
+ w.close();
+
+ Query filterQuery = new TermQuery(new Term("filter", "yes"));
+ Query query =
+ new BooleanQuery.Builder()
+ .add(filterQuery, Occur.FILTER)
+ .add(new PhraseQuery("optional", "quick", "brown"), Occur.SHOULD)
+ .add(new PhraseQuery("optional", "lazy", "dog"), Occur.SHOULD)
+ .setMinimumNumberShouldMatch(1)
+ .build();
+
+ IndexSearcher searcher = new IndexSearcher(reader);
+ LRUQueryCache queryCache =
+ new LRUQueryCache(16, 1 << 25, context -> context.reader() != null, 1f);
+ searcher.setQueryCache(queryCache);
+ searcher.setQueryCachingPolicy(
+ new QueryCachingPolicy() {
+ @Override
+ public void onUse(Query query) {}
+
+ @Override
+ public boolean shouldCache(Query query) {
+ return filterQuery.equals(query);
+ }
+ });
+
+ LeafReaderContext context = reader.leaves().get(0);
+ Weight filterWeight =
+ searcher.createWeight(searcher.rewrite(filterQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
+ ScorerSupplier filterScorerSupplier = filterWeight.scorerSupplier(context);
+ assertNotNull(filterScorerSupplier);
+ filterScorerSupplier.get(Long.MAX_VALUE).iterator().nextDoc();
+ assertEquals(1, queryCache.getCacheSize());
+
+ Weight weight =
+ searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f);
+ Scorer scorer = weight.scorerSupplier(context).get(Long.MAX_VALUE);
+ // The optional phrase disjunction makes the conjunction two-phase.
+ assertNotNull(TwoPhaseIterator.unwrap(scorer.iterator()));
+
+ BulkScorer bulkScorer = weight.bulkScorer(context);
+ assertThat(bulkScorer, instanceOf(ConstantScoreBulkScorer.class));
+
+ int[] collected = new int[expectedHitCount];
+ int[] count = new int[1];
+ LeafCollector collector =
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ collected[count[0]++] = doc;
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ int[] buffer = new int[32];
+ for (int size = stream.intoArray(buffer); size != 0; size = stream.intoArray(buffer)) {
+ System.arraycopy(buffer, 0, collected, count[0], size);
+ count[0] += size;
+ }
+ }
+ };
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, bulkScorer.score(collector, null, 0, maxDoc));
+ assertEquals(expectedHitCount, count[0]);
+ for (int i = 0; i < count[0]; ++i) {
+ assertEquals(1 + i * 50, collected[i]);
+ }
+
+ reader.close();
+ dir.close();
+ }
+
+ public void testFilterLeafCollectorPreservesCollectIntForDocIdStream() throws IOException {
+ FixedBitSet docs = new FixedBitSet(8);
+ docs.set(1);
+ docs.set(3);
+
+ int[] collected = new int[2];
+ int[] count = new int[1];
+ LeafCollector collector =
+ new FilterLeafCollector(
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ fail("Wrapped collector should not receive batch docs directly");
+ }
+ }) {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ collected[count[0]++] = doc;
+ }
+ };
+
+ collector.collect(new BitSetDocIdStream(docs, 0));
+ assertEquals(2, count[0]);
+ assertArrayEquals(new int[] {1, 3}, collected);
+ }
+
+ public void testFilterLeafCollectorPreservesCollectIntForRange() throws IOException {
+ int[] collected = new int[3];
+ int[] count = new int[1];
+ LeafCollector collector =
+ new FilterLeafCollector(
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ fail("Wrapped collector should not receive range docs directly");
+ }
+ }) {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ collected[count[0]++] = doc;
+ }
+ };
+
+ collector.collectRange(2, 5);
+ assertEquals(3, count[0]);
+ assertArrayEquals(new int[] {2, 3, 4}, collected);
+ }
+
+ public void testDefaultBulkScorerKeepsNoScoreConjunctionOnPerDocPath() throws IOException {
+ int[] docs = new int[20];
+ for (int i = 0; i < docs.length; ++i) {
+ docs[i] = 1 + i * 100;
+ }
+ CountingDocIdSetIterator lead = new CountingDocIdSetIterator(docs);
+ FixedBitSet filter = new FixedBitSet(9000);
+ filter.set(0, 9000);
+ Scorer leadScorer = new ConstantScoreScorer(0f, ScoreMode.COMPLETE_NO_SCORES, lead);
+ Scorer filterScorer =
+ new ConstantScoreScorer(
+ 0f, ScoreMode.COMPLETE_NO_SCORES, new BitDocIdSet(filter).iterator());
+ Scorer scorer =
+ new ConjunctionScorer(
+ Arrays.asList(leadScorer, filterScorer), java.util.Collections.emptyList());
+
+ int[] collected = new int[docs.length];
+ int[] count = new int[1];
+ LeafCollector collector =
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ collected[count[0]++] = doc;
+ }
+
+ @Override
+ public void collect(DocIdStream stream) {
+ fail("DefaultBulkScorer should not special-case no-score conjunctions");
+ }
+ };
+
+ BulkScorer bulkScorer = new DefaultBulkScorer(scorer);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, bulkScorer.score(collector, null, 0, 9000));
+ assertArrayEquals(docs, ArrayUtil.copyOfSubArray(collected, 0, count[0]));
+ assertEquals(0, lead.intoBitSetCalls);
+ }
+
+ public void testDefaultBulkScorerSkipsIntoBitSetForSparseNoScoreConjunction() throws IOException {
+ int[] docs = {1, 2, 4097, 8195};
+ CountingDocIdSetIterator lead = new CountingDocIdSetIterator(docs);
+ FixedBitSet filter = new FixedBitSet(9000);
+ filter.set(0, 9000);
+ filter.clear(1);
+ filter.clear(4097);
+ Scorer leadScorer = new ConstantScoreScorer(0f, ScoreMode.COMPLETE_NO_SCORES, lead);
+ Scorer filterScorer =
+ new ConstantScoreScorer(
+ 0f, ScoreMode.COMPLETE_NO_SCORES, new BitDocIdSet(filter).iterator());
+ Scorer scorer =
+ new ConjunctionScorer(
+ Arrays.asList(leadScorer, filterScorer), java.util.Collections.emptyList());
+
+ int[] collected = new int[2];
+ int[] count = new int[1];
+ LeafCollector collector =
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ collected[count[0]++] = doc;
+ }
+
+ @Override
+ public void collect(DocIdStream stream) {
+ fail("Sparse conjunctions should stay on the per-doc path");
+ }
+ };
+
+ BulkScorer bulkScorer = new DefaultBulkScorer(scorer);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, bulkScorer.score(collector, null, 0, 9000));
+ assertArrayEquals(new int[] {2, 8195}, ArrayUtil.copyOfSubArray(collected, 0, count[0]));
+ assertEquals(0, lead.intoBitSetCalls);
+ }
+
+ public void testDefaultBulkScorerSkipsIntoBitSetForNonBitSetConjunction() throws IOException {
+ int[] docs = new int[20];
+ for (int i = 0; i < docs.length; ++i) {
+ docs[i] = 1 + i * 100;
+ }
+ CountingDocIdSetIterator lead = new CountingDocIdSetIterator(docs);
+ CountingDocIdSetIterator other = new CountingDocIdSetIterator(docs);
+ Scorer leadScorer = new ConstantScoreScorer(0f, ScoreMode.COMPLETE_NO_SCORES, lead);
+ Scorer otherScorer = new ConstantScoreScorer(0f, ScoreMode.COMPLETE_NO_SCORES, other);
+ Scorer scorer =
+ new ConjunctionScorer(
+ Arrays.asList(leadScorer, otherScorer), java.util.Collections.emptyList());
+
+ int[] collected = new int[docs.length];
+ int[] count = new int[1];
+ LeafCollector collector =
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) {}
+
+ @Override
+ public void collect(int doc) {
+ collected[count[0]++] = doc;
+ }
+
+ @Override
+ public void collect(DocIdStream stream) {
+ fail("Non-bitset conjunctions should stay on the per-doc path");
+ }
+ };
+
+ BulkScorer bulkScorer = new DefaultBulkScorer(scorer);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, bulkScorer.score(collector, null, 0, 9000));
+ assertArrayEquals(docs, ArrayUtil.copyOfSubArray(collected, 0, count[0]));
+ assertEquals(0, lead.intoBitSetCalls);
+ assertEquals(0, other.intoBitSetCalls);
+ }
+
public void testConstantScoreBulkScorerRejectsScoreModeThatNeedsScores() {
expectThrows(
IllegalArgumentException.class,
@@ -372,9 +741,9 @@ public class TestBooleanScorer extends LuceneTestCase {
@Override
public int advance(int target) {
- do {
+ while (doc < target) {
nextDoc();
- } while (doc < target);
+ }
return doc;
}
@@ -384,12 +753,9 @@ public class TestBooleanScorer extends LuceneTestCase {
}
@Override
- public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) {
- intoBitSetCalls++;
- while (doc < upTo) {
- bitSet.set(doc - offset);
- nextDoc();
- }
+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
+ ++intoBitSetCalls;
+ super.intoBitSet(upTo, bitSet, offset);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java b/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java
index 68d4263b9f4..5caf7eecbd9 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
@@ -198,6 +199,16 @@ public class TestConjunctionDISI extends LuceneTestCase {
return set;
}
+ private static DocIdSetIterator bitSetConjunction(
+ FixedBitSet lead, FixedBitSet... bitSetFilters) {
+ List<DocIdSetIterator> iterators = new ArrayList<>();
+ iterators.add(anonymizeIterator(new BitDocIdSet(lead).iterator()));
+ for (FixedBitSet filter : bitSetFilters) {
+ iterators.add(new BitDocIdSet(filter).iterator());
+ }
+ return ConjunctionUtils.intersectIterators(iterators);
+ }
+
// Test that the conjunction iterator is correct
public void testConjunction() throws IOException {
final int iters = atLeast(100);
@@ -444,4 +455,166 @@ public class TestConjunctionDISI extends LuceneTestCase {
assertEquals(NO_MORE_DOCS, conjunction.nextDoc());
assertEquals(NO_MORE_DOCS, conjunction.docID());
}
+
+ public void testBitSetConjunctionIntoBitSet() throws IOException {
+ int maxDoc = 9000;
+ FixedBitSet lead = new FixedBitSet(maxDoc);
+ for (int doc : new int[] {1, 2, 4, 4097, 4100, 8190}) {
+ lead.set(doc);
+ }
+ FixedBitSet filter1 = new FixedBitSet(maxDoc);
+ for (int doc : new int[] {2, 4, 4100, 8190}) {
+ filter1.set(doc);
+ }
+ FixedBitSet filter2 = new FixedBitSet(maxDoc);
+ for (int doc : new int[] {2, 4100, 8190}) {
+ filter2.set(doc);
+ }
+
+ DocIdSetIterator conjunction = bitSetConjunction(lead, filter1, filter2);
+ assertEquals(2, conjunction.nextDoc());
+
+ FixedBitSet actual = new FixedBitSet(maxDoc);
+ actual.set(1234);
+ conjunction.intoBitSet(5000, actual, 0);
+
+ FixedBitSet expected = new FixedBitSet(maxDoc);
+ expected.set(2);
+ expected.set(1234);
+ expected.set(4100);
+ assertEquals(expected, actual);
+ assertEquals(8190, conjunction.docID());
+ assertEquals(NO_MORE_DOCS, conjunction.nextDoc());
+ }
+
+ public void testBitSetConjunctionIntoBitSetMultipleWindows() throws IOException {
+ int maxDoc = 9000;
+ FixedBitSet lead = new FixedBitSet(maxDoc);
+ FixedBitSet filter = new FixedBitSet(maxDoc);
+ for (int doc : new int[] {2, 4100, 8190}) {
+ lead.set(doc);
+ filter.set(doc);
+ }
+
+ DocIdSetIterator conjunction = bitSetConjunction(lead, filter);
+ assertEquals(2, conjunction.nextDoc());
+
+ FixedBitSet firstWindow = new FixedBitSet(4096);
+ conjunction.intoBitSet(4096, firstWindow, 0);
+ assertTrue(firstWindow.get(2));
+ assertEquals(1, firstWindow.cardinality());
+ assertEquals(4100, conjunction.docID());
+
+ FixedBitSet secondWindow = new FixedBitSet(4096);
+ conjunction.intoBitSet(8192, secondWindow, 4096);
+ assertTrue(secondWindow.get(4));
+ assertTrue(secondWindow.get(4094));
+ assertEquals(2, secondWindow.cardinality());
+ assertEquals(NO_MORE_DOCS, conjunction.docID());
+ }
+
+ public void testBitSetConjunctionIntoBitSetIgnoresLeadOnlyDocsPastDestination()
+ throws IOException {
+ FixedBitSet lead = new FixedBitSet(20);
+ lead.set(2);
+ lead.set(10);
+ FixedBitSet filter = new FixedBitSet(20);
+ filter.set(2);
+
+ DocIdSetIterator conjunction = bitSetConjunction(lead, filter);
+ assertEquals(2, conjunction.nextDoc());
+
+ FixedBitSet actual = new FixedBitSet(5);
+ conjunction.intoBitSet(20, actual, 0);
+ assertTrue(actual.get(2));
+ assertEquals(1, actual.cardinality());
+ assertEquals(NO_MORE_DOCS, conjunction.docID());
+ }
+
+ public void testBitSetConjunctionIntoBitSetUsesBulkPathWhenLeadCostIsHigh() throws IOException {
+ int maxDoc = 9000;
+ int[] docs = {2, 4, 4100, 8190};
+ CountingDocIdSetIterator lead = new CountingDocIdSetIterator(docs, 4096);
+ FixedBitSet filter = new FixedBitSet(maxDoc);
+ filter.set(0, maxDoc);
+
+ DocIdSetIterator conjunction =
+ ConjunctionUtils.intersectIterators(
+ Arrays.asList(lead, new BitDocIdSet(filter).iterator()));
+ assertEquals(2, conjunction.nextDoc());
+
+ FixedBitSet actual = new FixedBitSet(4096);
+ conjunction.intoBitSet(4096, actual, 0);
+ assertTrue(actual.get(2));
+ assertTrue(actual.get(4));
+ assertEquals(2, actual.cardinality());
+ assertEquals(1, lead.intoBitSetCalls);
+ assertEquals(4100, conjunction.docID());
+ }
+
+ public void testBitSetConjunctionIntoBitSetUsesPerDocPathWhenLeadCostIsLow() throws IOException {
+ int maxDoc = 9000;
+ int[] docs = {2, 4, 4100, 8190};
+ CountingDocIdSetIterator lead = new CountingDocIdSetIterator(docs, 4);
+ FixedBitSet filter = new FixedBitSet(maxDoc);
+ filter.set(0, maxDoc);
+
+ DocIdSetIterator conjunction =
+ ConjunctionUtils.intersectIterators(
+ Arrays.asList(lead, new BitDocIdSet(filter).iterator()));
+ assertEquals(2, conjunction.nextDoc());
+
+ FixedBitSet actual = new FixedBitSet(4096);
+ conjunction.intoBitSet(4096, actual, 0);
+ assertTrue(actual.get(2));
+ assertTrue(actual.get(4));
+ assertEquals(2, actual.cardinality());
+ assertEquals(0, lead.intoBitSetCalls);
+ assertEquals(4100, conjunction.docID());
+ }
+
+ private static class CountingDocIdSetIterator extends DocIdSetIterator {
+ private final int[] docs;
+ private final long cost;
+ private int index = -1;
+ private int doc = -1;
+ int intoBitSetCalls;
+
+ CountingDocIdSetIterator(int[] docs, long cost) {
+ this.docs = docs;
+ this.cost = cost;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() {
+ if (++index == docs.length) {
+ return doc = NO_MORE_DOCS;
+ }
+ return doc = docs[index];
+ }
+
+ @Override
+ public int advance(int target) {
+ while (doc < target) {
+ nextDoc();
+ }
+ return doc;
+ }
+
+ @Override
+ public long cost() {
+ return cost;
+ }
+
+ @Override
+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
+ ++intoBitSetCalls;
+ super.intoBitSet(upTo, bitSet, offset);
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
index c37a8c760fd..21c61ace753 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
@@ -28,6 +28,9 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.search.QueryUtils;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.FixedBitSet;
/**
* This class only tests some basic functionality in CSQ, the main parts are mostly tested by
@@ -143,6 +146,119 @@ public class TestConstantScoreQuery extends LuceneTestCase {
dir.close();
}
+ public void testConstantBulkScorerForwardsDocIdStream() throws IOException {
+ FixedBitSet docs = new FixedBitSet(8);
+ docs.set(1);
+ docs.set(3);
+ int[] collected = new int[2];
+ int[] count = new int[1];
+
+ BulkScorer scorer =
+ new ConstantScoreQuery.ConstantBulkScorer(
+ new BulkScorer() {
+ @Override
+ public int score(LeafCollector collector, Bits acceptDocs, int min, int max)
+ throws IOException {
+ collector.setScorer(scorable());
+ collector.collect(new BitSetDocIdStream(docs, 0));
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public long cost() {
+ return 2;
+ }
+ },
+ null,
+ 2f);
+
+ assertEquals(
+ DocIdSetIterator.NO_MORE_DOCS,
+ scorer.score(
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {
+ assertEquals(2f, scorer.score(), 0f);
+ }
+
+ @Override
+ public void collect(int doc) {
+ fail("ConstantBulkScorer should preserve DocIdStream collection");
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ int[] buffer = new int[collected.length];
+ for (int size = stream.intoArray(buffer);
+ size != 0;
+ size = stream.intoArray(buffer)) {
+ System.arraycopy(buffer, 0, collected, count[0], size);
+ count[0] += size;
+ }
+ }
+ },
+ null,
+ 0,
+ 8));
+ assertArrayEquals(new int[] {1, 3}, ArrayUtil.copyOfSubArray(collected, 0, count[0]));
+ }
+
+ public void testConstantBulkScorerForwardsCollectRange() throws IOException {
+ BulkScorer scorer =
+ new ConstantScoreQuery.ConstantBulkScorer(
+ new BulkScorer() {
+ @Override
+ public int score(LeafCollector collector, Bits acceptDocs, int min, int max)
+ throws IOException {
+ collector.setScorer(scorable());
+ collector.collectRange(2, 5);
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public long cost() {
+ return 3;
+ }
+ },
+ null,
+ 2f);
+
+ int[] range = new int[2];
+ assertEquals(
+ DocIdSetIterator.NO_MORE_DOCS,
+ scorer.score(
+ new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {
+ assertEquals(2f, scorer.score(), 0f);
+ }
+
+ @Override
+ public void collect(int doc) {
+ fail("ConstantBulkScorer should preserve range collection");
+ }
+
+ @Override
+ public void collectRange(int min, int max) {
+ range[0] = min;
+ range[1] = max;
+ }
+ },
+ null,
+ 0,
+ 8));
+ assertArrayEquals(new int[] {2, 5}, range);
+ }
+
+ private static Scorable scorable() {
+ return new Scorable() {
+ @Override
+ public float score() {
+ return 1f;
+ }
+ };
+ }
+
public void testRewriteBubblesUpMatchNoDocsQuery() throws IOException {
IndexSearcher searcher = newSearcher(new MultiReader());