You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2026/09/15 10:01:08 UTC
(lucene) branch main updated: Rewrite full-coverage doc-values range queries to FieldExistsQuery (#16582)
This is an automated email from the ASF dual-hosted git repository.
HoustonPutman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new a4d32c044e0 Rewrite full-coverage doc-values range queries to FieldExistsQuery (#16582)
a4d32c044e0 is described below
commit a4d32c044e06430dd9ad452cc41ff4f6550073ef
Author: Houston Putman <ho...@apache.org>
AuthorDate: Tue Sep 15 03:01:02 2026 -0700
Rewrite full-coverage doc-values range queries to FieldExistsQuery (#16582)
Also collapse IndexOrDocValuesQuery when both sides rewrite equally, and simplify
PointRangeQuery#canRewriteToMatchAllQuery.
---
lucene/CHANGES.txt | 4 ++
.../document/SortedNumericDocValuesRangeQuery.java | 11 +++--
.../lucene/search/IndexOrDocValuesQuery.java | 10 ++++
.../IndexSortSortedNumericDocValuesRangeQuery.java | 9 +++-
.../org/apache/lucene/search/PointRangeQuery.java | 13 +----
.../apache/lucene/search/TestDocValuesQueries.java | 23 +++++----
.../lucene/search/TestIndexOrDocValuesQuery.java | 30 ++++++++++++
...tIndexSortSortedNumericDocValuesRangeQuery.java | 56 +++++++++++++++++++---
8 files changed, 121 insertions(+), 35 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d0df2844676..00580cb83a1 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -427,6 +427,7 @@ Improvements
Optimizations
---------------------
+
* GITHUB#16593: DenseLiveDocs and SparseLiveDocs implement Bits#applyMask in bulk, restoring the
word-at-a-time masking that live docs had before GITHUB#13084 exposed them as LiveDocs rather than
as a FixedBitSet, so AcceptDocs, the bulk scorers and CheckIndex no longer test one bit per
@@ -504,6 +505,9 @@ Optimizations
* GITHUB#16588: AcceptDocs builds its bit set from the raw filter iterator so that
DocIdSetIterator#intoBitSet is used on segments with deletions. (John Maurice)
+* GITHUB#16582: Rewrite sorted-numeric doc-values range queries that cover every value of a field to a
+ FieldExistsQuery, and collapse an IndexOrDocValuesQuery whose two sides rewrite equally. (Houston Putman)
+
Bug Fixes
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
index 16ccd65c679..b4ac75288aa 100644
--- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
@@ -101,10 +101,13 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQuery
if (lowerValue > stats.max() || upperValue < stats.min()) {
return MatchNoDocsQuery.INSTANCE;
}
- if (lowerValue <= stats.min()
- && upperValue >= stats.max()
- && stats.docCount() == indexSearcher.getIndexReader().maxDoc()) {
- return MatchAllDocsQuery.INSTANCE;
+ if (lowerValue <= stats.min() && upperValue >= stats.max()) {
+ // Every value in the field falls within [lowerValue, upperValue], so this matches exactly
+ // the documents that have a value for the field.
+ if (stats.docCount() == indexSearcher.getIndexReader().maxDoc()) {
+ return MatchAllDocsQuery.INSTANCE;
+ }
+ return new FieldExistsQuery(field);
}
}
return super.rewrite(indexSearcher);
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java
index e4e975bb567..3dc56376c1b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java
@@ -121,6 +121,16 @@ public final class IndexOrDocValuesQuery extends Query {
|| dvRewrite.getClass() == MatchNoDocsQuery.class) {
return MatchNoDocsQuery.INSTANCE;
}
+ if (dvRewrite.getClass() == FieldExistsQuery.class) {
+ return dvRewrite;
+ } else if (indexRewrite.getClass() == FieldExistsQuery.class) {
+ return indexRewrite;
+ }
+ // If both sides rewrite to the same query there is no longer a choice to make between an
+ // index-based and a doc-values-based execution, so drop the wrapper.
+ if (indexRewrite.equals(dvRewrite)) {
+ return indexRewrite;
+ }
if (indexQuery != indexRewrite || dvQuery != dvRewrite) {
return new IndexOrDocValuesQuery(indexRewrite, dvRewrite);
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java
index 83e907dcd50..5de92f33c9f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java
@@ -133,8 +133,13 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends NumericDocValuesR
}
Query rewrittenFallback = fallbackQuery.rewrite(indexSearcher);
- if (rewrittenFallback.getClass() == MatchAllDocsQuery.class) {
- return MatchAllDocsQuery.INSTANCE;
+ // This query matches the same documents as its fallback, so if the fallback simplifies to a
+ // terminal query - match-all, match-none, or field-exists (e.g. a range covering every value of
+ // a sparse field) - the index-sort optimization adds nothing and we return it directly.
+ if (rewrittenFallback.getClass() == MatchAllDocsQuery.class
+ || rewrittenFallback.getClass() == MatchNoDocsQuery.class
+ || rewrittenFallback.getClass() == FieldExistsQuery.class) {
+ return rewrittenFallback;
}
if (rewrittenFallback == fallbackQuery) {
return this;
diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
index 84699ed81c8..4b349afe2b5 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
@@ -566,17 +566,8 @@ public abstract class PointRangeQuery extends Query {
};
}
- private boolean canRewriteToMatchAllQuery(IndexReader reader) throws IOException {
- for (LeafReaderContext context : reader.leaves()) {
- LeafReader leaf = context.reader();
- PointValues values = leaf.getPointValues(field);
-
- if (values == null || values.getDocCount() != leaf.maxDoc()) {
- return false;
- }
- }
-
- return true;
+ private boolean canRewriteToMatchAllQuery(IndexReader reader) {
+ return reader.maxDoc() == PointValues.getDocCount(reader, field);
}
private boolean canRewriteToFieldExistsQuery(IndexReader reader) {
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
index cb70682fece..2a1e192198e 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
@@ -980,12 +980,12 @@ public class TestDocValuesQueries extends LuceneTestCaseJupiter {
searcher.rewrite(
SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 250, 350)),
instanceOf(MatchNoDocsQuery.class));
- assertThat(
- searcher
- .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 174, 174))
- .getClass()
- .toString(),
- containsString("SortedNumericDocValuesRangeQuery"));
+ // A range that covers the field's only value but not every doc rewrites to a
+ // FieldExistsQuery (super_sparse has a value on a single doc).
+ assertEquals(
+ new FieldExistsQuery("super_sparse"),
+ searcher.rewrite(
+ SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 174, 174)));
assertThat(
searcher
.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 0, 150))
@@ -1004,12 +1004,11 @@ public class TestDocValuesQueries extends LuceneTestCaseJupiter {
.getClass()
.toString(),
containsString("SortedNumericDocValuesRangeQuery"));
- assertThat(
- searcher
- .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 250))
- .getClass()
- .toString(),
- containsString("SortedNumericDocValuesRangeQuery"));
+ // A range that covers every value of a sparse field (a value on all but one doc) rewrites
+ // to a FieldExistsQuery rather than staying a range query.
+ assertEquals(
+ new FieldExistsQuery("sparse"),
+ searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 250)));
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.java
index dc62b192ac3..176494251ba 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.java
@@ -264,4 +264,34 @@ public class TestIndexOrDocValuesQuery extends LuceneTestCase {
}
}
}
+
+ public void testRewriteToFieldExistsWhenBothSidesCoverAllValues() throws Exception {
+ var config = newIndexWriterConfig().setCodec(TestUtil.getDefaultCodec());
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, config)) {
+ for (int i = 0; i < 10; ++i) {
+ Document doc = new Document();
+ // sparse field: a value on every doc but one, so it can't rewrite to MatchAllDocsQuery
+ if (i != 5) {
+ doc.add(new LongPoint("sparse", 100 + i));
+ doc.add(SortedNumericDocValuesField.indexedField("sparse", 100 + i));
+ }
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+
+ try (IndexReader reader = DirectoryReader.open(w)) {
+ IndexSearcher searcher = newSearcher(reader);
+ // A range covering every value of the sparse field: both the point and the doc-values side
+ // rewrite to a FieldExistsQuery, so the wrapper collapses to a single FieldExistsQuery
+ // rather than re-wrapping two identical queries.
+ IndexOrDocValuesQuery query =
+ new IndexOrDocValuesQuery(
+ LongPoint.newRangeQuery("sparse", 0, 250),
+ SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 250));
+ QueryUtils.check(random(), query, searcher);
+ assertEquals(new FieldExistsQuery("sparse"), query.rewrite(searcher));
+ }
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java
index 69b667241c0..1885f7480a2 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java
@@ -31,6 +31,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
@@ -412,24 +413,67 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCas
dir.close();
}
- public void testRewriteFallbackQuery() throws IOException {
+ public void testRewriteFallbackToMatchNone() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
writer.addDocument(new Document());
IndexReader reader = writer.getReader();
- // Create an (unrealistic) fallback query that is sure to be rewritten.
+ // An empty BooleanQuery rewrites to a MatchNoDocsQuery; the wrapper matches the same docs as
+ // its
+ // fallback, so it collapses to that MatchNoDocsQuery instead of re-wrapping it.
Query fallbackQuery = new BooleanQuery.Builder().build();
Query query = new IndexSortSortedNumericDocValuesRangeQuery("field", 1, 42, fallbackQuery);
+ assertEquals(new MatchNoDocsQuery(), query.rewrite(newSearcher(reader)));
+
+ writer.close();
+ reader.close();
+ dir.close();
+ }
+
+ public void testRewriteFallbackReWrapped() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ writer.addDocument(new Document());
+ IndexReader reader = writer.getReader();
+
+ // A fallback that rewrites to a non-terminal query (BoostQuery with boost 1 unwraps to its
+ // inner query) keeps the index-sort wrapper, now wrapping the rewritten fallback.
+ TermQuery inner = new TermQuery(new Term("field", "x"));
+ Query fallbackQuery = new BoostQuery(inner, 1.0f);
+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field", 1, 42, fallbackQuery);
Query rewrittenQuery = query.rewrite(newSearcher(reader));
- assertNotEquals(query, rewrittenQuery);
MatcherAssert.assertThat(
rewrittenQuery, instanceOf(IndexSortSortedNumericDocValuesRangeQuery.class));
+ assertEquals(
+ inner, ((IndexSortSortedNumericDocValuesRangeQuery) rewrittenQuery).getFallbackQuery());
+
+ writer.close();
+ reader.close();
+ dir.close();
+ }
+
+ public void testRewriteFallbackToFieldExists() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+ // Indexed (skip-indexed) sorted-numeric field on all docs but one, so the field is sparse and
+ // its values span [100, 108].
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ if (i != 5) {
+ doc.add(SortedNumericDocValuesField.indexedField("field", 100 + i));
+ }
+ writer.addDocument(doc);
+ }
+ IndexReader reader = writer.getReader();
- IndexSortSortedNumericDocValuesRangeQuery rangeQuery =
- (IndexSortSortedNumericDocValuesRangeQuery) rewrittenQuery;
- assertEquals(MatchNoDocsQuery.INSTANCE, rangeQuery.getFallbackQuery());
+ // A range covering every value of the sparse field: the fallback rewrites to a
+ // FieldExistsQuery,
+ // which the index-sort query must return directly rather than re-wrapping.
+ Query fallbackQuery = SortedNumericDocValuesField.newSlowRangeQuery("field", 0, 250);
+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field", 0, 250, fallbackQuery);
+ assertEquals(new FieldExistsQuery("field"), query.rewrite(newSearcher(reader)));
writer.close();
reader.close();