You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ta...@apache.org on 2026/08/13 18:32:07 UTC
(lucene) 01/02: Add bulk-scorer vs doc-by-doc differential oracle test
This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch bulk-scorer-oracle
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 8a58f3f451a6c16e61588883e3ec77baa8c6ced5
Author: tballison <ti...@elastic.co>
AuthorDate: Tue Aug 11 11:27:57 2026 -0400
Add bulk-scorer vs doc-by-doc differential oracle test
---
.../test/org/apache/lucene/search/FieldSpec.java | 42 ++++
.../apache/lucene/search/RandomIndexBuilder.java | 136 +++++++++++
.../lucene/search/RandomLuceneQueryGenerator.java | 250 +++++++++++++++++++++
.../lucene/search/TestBulkScorerConsistency.java | 231 +++++++++++++++++++
4 files changed, 659 insertions(+)
diff --git a/lucene/core/src/test/org/apache/lucene/search/FieldSpec.java b/lucene/core/src/test/org/apache/lucene/search/FieldSpec.java
new file mode 100644
index 00000000000..c2b007bd588
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/FieldSpec.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.util.List;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Shared schema description used by {@link RandomIndexBuilder} (write side) and {@link
+ * RandomLuceneQueryGenerator} (query side).
+ */
+record FieldSpec(String name, FieldKind kind, List<BytesRef> terms, long minValue, long maxValue) {
+
+ enum FieldKind {
+ INDEXED_KEYWORD,
+ DV_KEYWORD,
+ DV_NUMERIC,
+ POINT_LONG,
+ }
+
+ static FieldSpec keyword(String name, FieldKind kind, List<BytesRef> terms) {
+ return new FieldSpec(name, kind, terms, 0, 0);
+ }
+
+ static FieldSpec numeric(String name, FieldKind kind, long minValue, long maxValue) {
+ return new FieldSpec(name, kind, List.of(), minValue, maxValue);
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/RandomIndexBuilder.java b/lucene/core/src/test/org/apache/lucene/search/RandomIndexBuilder.java
new file mode 100644
index 00000000000..9bca090cfa6
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/RandomIndexBuilder.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.FieldSpec.FieldKind;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Builds a random Lucene index whose doc counts bracket the thresholds that gate bulk-scorer
+ * specializations: the 4096-doc windows of the doc-values skip index, {@code
+ * DenseConjunctionBulkScorer} and {@code BooleanScorer}, and the 1/32 density cutoff.
+ */
+class RandomIndexBuilder {
+
+ static final int[] INTERESTING_DOC_COUNTS = {
+ 1, 512, 1024, 2047, 2048, 2049, 4095, 4096, 4097, 8192
+ };
+
+ private final Random random;
+
+ RandomIndexBuilder(Random random) {
+ this.random = random;
+ }
+
+ List<FieldSpec> build(Directory dir) throws IOException {
+ int numDocs = chooseDocCount();
+ int lowCard = 2 + random.nextInt(6);
+ int highCard = Math.max(lowCard + 1, numDocs / 4);
+ List<FieldSpec> specs = buildSpecs(lowCard, highCard);
+
+ try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) {
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ for (FieldSpec spec : specs) {
+ addFieldValue(doc, spec, i);
+ }
+ w.addDocument(doc);
+ }
+
+ if (random.nextBoolean() && numDocs > 1) {
+ int toDelete = 1 + random.nextInt(Math.min(numDocs / 4, 10));
+ for (int i = 0; i < toDelete; i++) {
+ w.deleteDocuments(
+ new Term(
+ specs.get(0).name(),
+ specs.get(0).terms().get(random.nextInt(specs.get(0).terms().size()))));
+ }
+ }
+
+ if (random.nextBoolean()) {
+ w.forceMerge(1);
+ }
+ }
+
+ return specs;
+ }
+
+ private int chooseDocCount() {
+ if (random.nextInt(3) == 0) {
+ return INTERESTING_DOC_COUNTS[random.nextInt(INTERESTING_DOC_COUNTS.length)];
+ }
+ return 1 + random.nextInt(8192);
+ }
+
+ private List<FieldSpec> buildSpecs(int lowCard, int highCard) {
+ List<FieldSpec> specs = new ArrayList<>();
+ specs.add(FieldSpec.keyword("f_kw", FieldKind.INDEXED_KEYWORD, makeTerms("kw", lowCard)));
+ specs.add(FieldSpec.keyword("f_dv_kw", FieldKind.DV_KEYWORD, makeTerms("dv", lowCard)));
+ specs.add(FieldSpec.numeric("f_dv_num", FieldKind.DV_NUMERIC, 0L, highCard - 1L));
+ specs.add(FieldSpec.numeric("f_pt", FieldKind.POINT_LONG, 0L, highCard - 1L));
+ return specs;
+ }
+
+ private List<BytesRef> makeTerms(String prefix, int n) {
+ List<BytesRef> terms = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ terms.add(new BytesRef(prefix + "_" + i));
+ }
+ return terms;
+ }
+
+ private void addFieldValue(Document doc, FieldSpec spec, int docId) {
+ switch (spec.kind()) {
+ case INDEXED_KEYWORD -> {
+ BytesRef term = spec.terms().get(docId % spec.terms().size());
+ doc.add(new StringField(spec.name(), term.utf8ToString(), Field.Store.NO));
+ }
+ case DV_KEYWORD -> {
+ BytesRef term = spec.terms().get(docId % spec.terms().size());
+ if (random.nextInt(5) != 0) {
+ // indexedField enables the skip index, required for the block-aware range iterators
+ doc.add(SortedSetDocValuesField.indexedField(spec.name(), term));
+ }
+ }
+ case DV_NUMERIC -> {
+ long val = spec.minValue() + (docId % (spec.maxValue() - spec.minValue() + 1));
+ if (random.nextInt(5) != 0) {
+ doc.add(new SortedNumericDocValuesField(spec.name(), val));
+ }
+ }
+ case POINT_LONG -> {
+ long val = spec.minValue() + (docId % (spec.maxValue() - spec.minValue() + 1));
+ doc.add(new LongPoint(spec.name() + "_point", val));
+ doc.add(new SortedNumericDocValuesField(spec.name() + "_dv", val));
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/RandomLuceneQueryGenerator.java b/lucene/core/src/test/org/apache/lucene/search/RandomLuceneQueryGenerator.java
new file mode 100644
index 00000000000..fca15b3b161
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/RandomLuceneQueryGenerator.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Generates random Lucene {@link Query} trees for differential oracle testing.
+ *
+ * <p>Uses only the Lucene public API. Driven by a caller-supplied {@link Random}; a failing seed
+ * replays exactly. Generates nested {@link BooleanQuery} nodes mixing all four {@link Occur}
+ * values, doc-values-only leaf queries (which produce {@link TwoPhaseIterator}s), indexed queries,
+ * {@link IndexOrDocValuesQuery}, and trivial sentinels.
+ */
+// newSlowExactQuery / newSlowRangeQuery are Lucene test-only APIs marked deprecated.
+@SuppressWarnings("deprecation")
+class RandomLuceneQueryGenerator {
+
+ static final int MAX_DEPTH = 3;
+ static final int MAX_CLAUSES = 5;
+
+ private final Random random;
+ private final List<FieldSpec> fields;
+
+ RandomLuceneQueryGenerator(Random random, List<FieldSpec> fields) {
+ this.random = random;
+ this.fields = fields;
+ }
+
+ Query next() {
+ // ~1/3 of top-level queries are pure FILTER conjunctions. This biases coverage toward
+ // DenseConjunctionBulkScorer, which only fires for all-FILTER queries on large dense indices.
+ if (random.nextInt(3) == 0) {
+ return randomFilterConjunction();
+ }
+ return generate(MAX_DEPTH);
+ }
+
+ /** Generates a BooleanQuery where every clause uses {@link Occur#FILTER}. */
+ private Query randomFilterConjunction() {
+ int numClauses = 2 + random.nextInt(MAX_CLAUSES - 1);
+ BooleanQuery.Builder b = new BooleanQuery.Builder();
+ for (int i = 0; i < numClauses; i++) {
+ b.add(randomLeaf(), Occur.FILTER);
+ }
+ return b.build();
+ }
+
+ private Query generate(int depth) {
+ if (depth == 0 || (depth < MAX_DEPTH && random.nextInt(10) < 3)) {
+ return randomLeaf();
+ }
+ return randomBool(depth);
+ }
+
+ private Query randomBool(int depth) {
+ int numClauses = 1 + random.nextInt(MAX_CLAUSES);
+ BooleanQuery.Builder b = new BooleanQuery.Builder();
+
+ int mustNotCount = 0;
+ List<Query> clauses = new ArrayList<>(numClauses);
+ List<Occur> occurs = new ArrayList<>(numClauses);
+
+ for (int i = 0; i < numClauses; i++) {
+ clauses.add(generate(depth - 1));
+ occurs.add(randomOccur());
+ if (occurs.get(i) == Occur.MUST_NOT) {
+ mustNotCount++;
+ }
+ }
+
+ if (mustNotCount == numClauses) {
+ occurs.set(random.nextInt(numClauses), Occur.FILTER);
+ }
+
+ int numShould = 0;
+ for (int i = 0; i < numClauses; i++) {
+ b.add(clauses.get(i), occurs.get(i));
+ if (occurs.get(i) == Occur.SHOULD) {
+ numShould++;
+ }
+ }
+
+ if (numShould > 0) {
+ b.setMinimumNumberShouldMatch(random.nextInt(numShould + 1));
+ }
+
+ return b.build();
+ }
+
+ private Occur randomOccur() {
+ return switch (random.nextInt(4)) {
+ case 0 -> Occur.MUST;
+ case 1 -> Occur.FILTER;
+ case 2 -> Occur.SHOULD;
+ default -> Occur.MUST_NOT;
+ };
+ }
+
+ private Query randomLeaf() {
+ int roll = random.nextInt(20);
+ if (roll == 0) return new MatchAllDocsQuery();
+ if (roll == 1) return new MatchNoDocsQuery();
+ if (roll == 2) return new ConstantScoreQuery(randomLeafFromField());
+ return randomLeafFromField();
+ }
+
+ private Query randomLeafFromField() {
+ FieldSpec f = fields.get(random.nextInt(fields.size()));
+ return switch (f.kind()) {
+ case INDEXED_KEYWORD -> randomIndexedKeywordLeaf(f);
+ case DV_KEYWORD -> randomDvKeywordLeaf(f);
+ case DV_NUMERIC -> randomDvNumericLeaf(f);
+ case POINT_LONG -> randomPointLongLeaf(f);
+ };
+ }
+
+ private Query randomIndexedKeywordLeaf(FieldSpec f) {
+ return switch (random.nextInt(3)) {
+ case 0 -> new TermQuery(new Term(f.name(), randomTerm(f)));
+ case 1 -> {
+ int n = 1 + random.nextInt(Math.min(3, f.terms().size()));
+ List<BytesRef> set = new ArrayList<>(n);
+ for (int i = 0; i < n; i++) {
+ set.add(randomTerm(f));
+ }
+ yield new TermInSetQuery(f.name(), set);
+ }
+ default -> {
+ BytesRef lo = randomTermOrNull(f);
+ BytesRef hi = randomTermOrNull(f);
+ if (lo != null && hi != null && lo.compareTo(hi) > 0) {
+ BytesRef tmp = lo;
+ lo = hi;
+ hi = tmp;
+ }
+ yield new TermRangeQuery(f.name(), lo, hi, random.nextBoolean(), random.nextBoolean());
+ }
+ };
+ }
+
+ private Query randomDvKeywordLeaf(FieldSpec f) {
+ return switch (random.nextInt(3)) {
+ case 0 -> SortedSetDocValuesField.newSlowExactQuery(f.name(), randomTerm(f));
+ case 1 -> {
+ BytesRef lo = randomTerm(f);
+ BytesRef hi = randomTerm(f);
+ if (lo.compareTo(hi) > 0) {
+ BytesRef tmp = lo;
+ lo = hi;
+ hi = tmp;
+ }
+ yield SortedSetDocValuesField.newSlowRangeQuery(f.name(), lo, hi, true, true);
+ }
+ default -> randomDvKeywordSetLeaf(f);
+ };
+ }
+
+ /**
+ * Picks every other term so there is always an ordinal gap, forcing the non-contiguous ordinal
+ * set iterator whose {@code docIDRunEnd()} had the GH#16450 bug.
+ */
+ private Query randomDvKeywordSetLeaf(FieldSpec f) {
+ List<BytesRef> all = f.terms();
+ if (all.size() < 3) {
+ // Can't form a non-contiguous set with fewer than 3 terms; fall back to exact.
+ return SortedSetDocValuesField.newSlowExactQuery(f.name(), randomTerm(f));
+ }
+ int start = random.nextInt(2); // vary the skip pattern across seeds
+ List<BytesRef> subset = new ArrayList<>();
+ for (int i = start; i < all.size(); i += 2) {
+ subset.add(all.get(i));
+ }
+ if (subset.size() < 2) {
+ subset.add(all.get(all.size() - 1));
+ }
+ return SortedSetDocValuesField.newSlowSetQuery(f.name(), subset);
+ }
+
+ private Query randomDvNumericLeaf(FieldSpec f) {
+ if (random.nextBoolean()) {
+ return SortedNumericDocValuesField.newSlowExactQuery(f.name(), randomLong(f));
+ }
+ long lo = randomLong(f);
+ long hi = randomLong(f);
+ if (lo > hi) {
+ long tmp = lo;
+ lo = hi;
+ hi = tmp;
+ }
+ return SortedNumericDocValuesField.newSlowRangeQuery(f.name(), lo, hi);
+ }
+
+ private Query randomPointLongLeaf(FieldSpec f) {
+ long lo = randomLong(f);
+ long hi = randomLong(f);
+ if (lo > hi) {
+ long tmp = lo;
+ lo = hi;
+ hi = tmp;
+ }
+ Query pointQuery = LongPoint.newRangeQuery(f.name() + "_point", lo, hi);
+ Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(f.name() + "_dv", lo, hi);
+ if (random.nextBoolean()) {
+ return new IndexOrDocValuesQuery(pointQuery, dvQuery);
+ }
+ return random.nextBoolean() ? pointQuery : dvQuery;
+ }
+
+ private BytesRef randomTerm(FieldSpec f) {
+ if (random.nextInt(5) == 0 || f.terms().isEmpty()) {
+ return new BytesRef("zzz_outside_" + random.nextInt(100));
+ }
+ return f.terms().get(random.nextInt(f.terms().size()));
+ }
+
+ private BytesRef randomTermOrNull(FieldSpec f) {
+ return random.nextInt(5) == 0 ? null : randomTerm(f);
+ }
+
+ private long randomLong(FieldSpec f) {
+ if (random.nextInt(5) == 0) {
+ return f.maxValue() + 1 + random.nextInt(10);
+ }
+ long range = f.maxValue() - f.minValue() + 1;
+ return f.minValue() + (range == 0 ? 0 : (random.nextLong() & Long.MAX_VALUE) % range);
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBulkScorerConsistency.java b/lucene/core/src/test/org/apache/lucene/search/TestBulkScorerConsistency.java
new file mode 100644
index 00000000000..d9d7c727739
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBulkScorerConsistency.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.search.FixedBitSetCollector;
+import org.apache.lucene.tests.search.ScorerIndexSearcher;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+
+/**
+ * Differential oracle: the optimised bulk-scoring path must produce the same document set as the
+ * doc-by-doc {@link Scorer} path ({@link ScorerIndexSearcher}) for random queries over random
+ * indexes. Compares match sets only, not scores. Written after the {@code
+ * DocValuesRangeIterator.docIDRunEnd()} bug (GH#16450), which this harness catches.
+ */
+public class TestBulkScorerConsistency extends LuceneTestCase {
+
+ /**
+ * Random index, random queries, fast == slow. The fast side is a bare {@link IndexSearcher} so
+ * the production bulk scorers run unwrapped by the test framework.
+ */
+ public void testBulkEqualsScorer() throws IOException {
+ try (Directory dir = newDirectory()) {
+ List<FieldSpec> fields = new RandomIndexBuilder(random()).build(dir);
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher fastSearcher = new IndexSearcher(reader);
+ ScorerIndexSearcher slowSearcher = new ScorerIndexSearcher(reader);
+ fastSearcher.setQueryCache(null);
+ slowSearcher.setQueryCache(null);
+
+ RandomLuceneQueryGenerator gen = new RandomLuceneQueryGenerator(random(), fields);
+ int numQueries = 20 + random().nextInt(81);
+ for (int q = 0; q < numQueries; q++) {
+ assertBulkEqualsScorer(fastSearcher, slowSearcher, gen.next());
+ }
+ }
+ }
+ }
+
+ /** Self-test: a bulk scorer that collects without checking matches must be flagged. */
+ public void testHarnessDetectsBrokenBulkScorer() throws IOException {
+ try (Directory dir = newDirectory()) {
+ try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) {
+ for (int i = 0; i < 100; i++) {
+ Document doc = new Document();
+ doc.add(new StringField("f", i % 2 == 0 ? "even" : "odd", Field.Store.NO));
+ w.addDocument(doc);
+ }
+ }
+
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ ScorerIndexSearcher correct = new ScorerIndexSearcher(reader);
+ LyingIndexSearcher lying = new LyingIndexSearcher(reader);
+ correct.setQueryCache(null);
+ lying.setQueryCache(null);
+
+ int maxDoc = reader.maxDoc();
+ Query query = new TermQuery(new Term("f", "even"));
+
+ FixedBitSet correctResult =
+ correct.search(query, FixedBitSetCollector.createManager(maxDoc));
+ FixedBitSet lyingResult = lying.search(query, FixedBitSetCollector.createManager(maxDoc));
+
+ assertEquals(50, correctResult.cardinality());
+ assertFalse(lyingResult.equals(correctResult));
+ assertEquals(maxDoc, lyingResult.cardinality());
+ }
+ }
+ }
+
+ /**
+ * Regression test for GH#16450: {@code docIDRunEnd()} over-reported for a non-contiguous ordinal
+ * set, so on a {@code WINDOW_SIZE + 1}-doc index the trailing single-doc window looked fully
+ * matching and {@code DenseConjunctionBulkScorer} collected it via {@code collectRange} without
+ * confirming {@code matches()}.
+ */
+ public void testDVOrdinalSetFalsePositive() throws IOException {
+ int maxDoc = DenseConjunctionBulkScorer.WINDOW_SIZE + 1;
+ try (Directory dir = newDirectory()) {
+ try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) {
+ for (int i = 0; i < maxDoc; i++) {
+ Document doc = new Document();
+ // Most docs: "aaa" (ord 0). Every 100th: "ccc" (ord 2) to establish the 3-term vocab.
+ // Last doc: "bbb" (ord 1) — inside bounding range [0,2] but NOT in set {0,2}.
+ String val = (i == maxDoc - 1) ? "bbb" : (i % 100 == 0 ? "ccc" : "aaa");
+ doc.add(SortedDocValuesField.indexedField("dv", new BytesRef(val)));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ }
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher fast = new IndexSearcher(reader);
+ ScorerIndexSearcher slow = new ScorerIndexSearcher(reader);
+ fast.setQueryCache(null);
+ slow.setQueryCache(null);
+
+ // Ordinal set {aaa=0, ccc=2} with gap at bbb=1: non-contiguous, so the block iterator
+ // cannot prove whole-block matches and docIDRunEnd() must stay conservative.
+ Query setQuery =
+ SortedDocValuesField.newSlowSetQuery(
+ "dv", List.of(new BytesRef("aaa"), new BytesRef("ccc")));
+ Query rangeQuery =
+ SortedDocValuesField.newSlowRangeQuery(
+ "dv", new BytesRef("aaa"), new BytesRef("ccc"), true, true);
+ Query q =
+ new BooleanQuery.Builder()
+ .add(setQuery, BooleanClause.Occur.FILTER)
+ .add(rangeQuery, BooleanClause.Occur.FILTER)
+ .build();
+
+ FixedBitSet fastBits = fast.search(q, FixedBitSetCollector.createManager(maxDoc));
+ FixedBitSet slowBits = slow.search(q, FixedBitSetCollector.createManager(maxDoc));
+ assertEquals("fast path cardinality", maxDoc - 1, fastBits.cardinality());
+ assertEquals("slow path cardinality", maxDoc - 1, slowBits.cardinality());
+ }
+ }
+ }
+
+ static void assertBulkEqualsScorer(
+ IndexSearcher fastSearcher, ScorerIndexSearcher slowSearcher, Query query)
+ throws IOException {
+ int maxDoc = fastSearcher.getIndexReader().maxDoc();
+ FixedBitSet fast = fastSearcher.search(query, FixedBitSetCollector.createManager(maxDoc));
+ FixedBitSet slow = slowSearcher.search(query, FixedBitSetCollector.createManager(maxDoc));
+ if (slow.equals(fast) == false) {
+ StringBuilder diff = new StringBuilder();
+ for (int doc = 0, shown = 0; doc < maxDoc && shown < 10; doc++) {
+ if (slow.get(doc) != fast.get(doc)) {
+ diff.append(doc).append(slow.get(doc) ? " missed by bulk; " : " bulk false positive; ");
+ shown++;
+ }
+ }
+ fail("bulk scorer and doc-by-doc scorer disagree on: " + query + " — docs: " + diff);
+ }
+ }
+
+ static class LyingIndexSearcher extends IndexSearcher {
+ LyingIndexSearcher(org.apache.lucene.index.IndexReader r) {
+ super(r);
+ }
+
+ @Override
+ protected void searchLeaf(
+ LeafReaderContext ctx, int minDocId, int maxDocId, Weight weight, Collector collector)
+ throws IOException {
+ final LeafCollector leafCollector;
+ try {
+ leafCollector = collector.getLeafCollector(ctx);
+ } catch (CollectionTerminatedException _) {
+ return;
+ }
+ ScorerSupplier ss = weight.scorerSupplier(ctx);
+ if (ss == null) {
+ leafCollector.finish();
+ return;
+ }
+ BulkScorer real = ss.bulkScorer();
+ if (real == null) {
+ leafCollector.finish();
+ return;
+ }
+ try {
+ int maxDoc = Math.min(maxDocId, ctx.reader().maxDoc());
+ new LyingBulkScorer(real)
+ .score(leafCollector, ctx.reader().getLiveDocs(), minDocId, maxDoc);
+ } catch (CollectionTerminatedException _) {
+ // normal
+ }
+ leafCollector.finish();
+ }
+ }
+
+ static class LyingBulkScorer extends BulkScorer {
+ private final BulkScorer in;
+
+ LyingBulkScorer(BulkScorer in) {
+ this.in = in;
+ }
+
+ @Override
+ public int score(LeafCollector collector, Bits acceptDocs, int min, int max)
+ throws IOException {
+ collector.setScorer(
+ new Scorable() {
+ @Override
+ public float score() {
+ return 0f;
+ }
+ });
+ for (int doc = min; doc < max; doc++) {
+ if (acceptDocs == null || acceptDocs.get(doc)) {
+ collector.collect(doc);
+ }
+ }
+ return max;
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+ }
+}