You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2025/08/19 08:12:05 UTC
(lucene) branch branch_10x updated: Rewrite SortedNumericDocValuesRangeQuery using DocValuesSkippers (#15083)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 37aa3ee363e Rewrite SortedNumericDocValuesRangeQuery using DocValuesSkippers (#15083)
37aa3ee363e is described below
commit 37aa3ee363edc7950e6b34d8ad981cdd0f7d64ed
Author: Alan Woodward <ro...@apache.org>
AuthorDate: Tue Aug 19 08:57:18 2025 +0100
Rewrite SortedNumericDocValuesRangeQuery using DocValuesSkippers (#15083)
If DocValuesSkippers are enabled on a field, we can check at rewrite time
if a range query will match all or none of the documents in all segments,
and rewrite to a corresponding MatchAllDocsQuery or MatchNoDocsQuery.
---
lucene/CHANGES.txt | 2 +
.../document/SortedNumericDocValuesRangeQuery.java | 12 ++++
.../org/apache/lucene/index/DocValuesSkipper.java | 64 ++++++++++++++++++
.../apache/lucene/search/TestDocValuesQueries.java | 75 ++++++++++++++++++++++
4 files changed, 153 insertions(+)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index be8998b6e2d..709f4dd9873 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -148,6 +148,8 @@ Optimizations
* GITHUB#15039: Score computations are now more reliably vectorized.
(Adrien Grand, Guo Feng)
+* GITHUB#15083: Use DocValuesSkippers in SortedNumericDocValuesRangeQuery#rewrite(). (Alan Woodward)
+
Changes in Runtime Behavior
---------------------
* GITHUB#14823: Decrease TieredMergePolicy's default number of segments per
diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
index 1965e1cde35..e72c0f81061 100644
--- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
@@ -31,6 +31,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.DocValuesRangeIterator;
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
@@ -97,6 +98,17 @@ final class SortedNumericDocValuesRangeQuery extends Query {
if (lowerValue > upperValue) {
return new MatchNoDocsQuery();
}
+ long globalMin = DocValuesSkipper.globalMinValue(indexSearcher, field);
+ long globalMax = DocValuesSkipper.globalMaxValue(indexSearcher, field);
+ if (lowerValue > globalMax || upperValue < globalMin) {
+ return new MatchNoDocsQuery();
+ }
+ if (lowerValue <= globalMin
+ && upperValue >= globalMax
+ && DocValuesSkipper.globalDocCount(indexSearcher, field)
+ == indexSearcher.getIndexReader().maxDoc()) {
+ return new MatchAllDocsQuery();
+ }
return super.rewrite(indexSearcher);
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java
index 591bfacd141..6f1c06bb608 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
/**
* Skipper for {@link DocValues}.
@@ -122,4 +123,67 @@ public abstract class DocValuesSkipper {
advance(maxDocID + 1);
}
}
+
+ /**
+ * Returns the minimum value for a field across all segments, or {@link Long#MIN_VALUE} if not
+ * available
+ *
+ * @param searcher a searcher over the index
+ * @param field the field to retrieve values for
+ */
+ public static long globalMinValue(IndexSearcher searcher, String field) throws IOException {
+ long minValue = Long.MAX_VALUE;
+ for (LeafReaderContext ctx : searcher.getLeafContexts()) {
+ if (ctx.reader().getFieldInfos().fieldInfo(field) == null) {
+ continue; // no field values in this segment, so we can ignore it
+ }
+ DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper(field);
+ if (skipper == null) {
+ minValue = Long.MIN_VALUE;
+ } else {
+ minValue = Math.min(minValue, skipper.minValue());
+ }
+ }
+ return minValue;
+ }
+
+ /**
+ * Returns the maximum value for a field across all segments, or {@link Long#MIN_VALUE} if not
+ * available
+ *
+ * @param searcher a searcher over the index
+ * @param field the field to retrieve values for
+ */
+ public static long globalMaxValue(IndexSearcher searcher, String field) throws IOException {
+ long maxValue = Long.MIN_VALUE;
+ for (LeafReaderContext ctx : searcher.getLeafContexts()) {
+ if (ctx.reader().getFieldInfos().fieldInfo(field) == null) {
+ continue; // no field values in this segment, so we can ignore it
+ }
+ DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper(field);
+ if (skipper == null) {
+ maxValue = Long.MAX_VALUE;
+ } else {
+ maxValue = Math.max(maxValue, skipper.maxValue());
+ }
+ }
+ return maxValue;
+ }
+
+ /**
+ * Returns the total skipper document count for a field across all segments
+ *
+ * @param searcher a searcher over the index
+ * @param field the field to retrieve values for
+ */
+ public static int globalDocCount(IndexSearcher searcher, String field) throws IOException {
+ int docCount = 0;
+ for (LeafReaderContext ctx : searcher.getLeafContexts()) {
+ DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper(field);
+ if (skipper != null) {
+ docCount += skipper.docCount();
+ }
+ }
+ return docCount;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
index f081246e7eb..4badfc294fc 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
@@ -16,6 +16,9 @@
*/
package org.apache.lucene.search;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -648,4 +651,76 @@ public class TestDocValuesQueries extends LuceneTestCase {
Weight w = searcher.createWeight(query, ScoreMode.COMPLETE, 1.0f);
assertEquals(expectedCount, w.count(searcher.reader.leaves().getFirst()));
}
+
+ public void testSortedNumericDocValuesRangeQueryRewrites() throws Exception {
+ try (Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
+ for (int i = 0; i < 100; i++) {
+ Document doc = new Document();
+ doc.add(SortedNumericDocValuesField.indexedField("with_index", 100 + i));
+ doc.add(new SortedNumericDocValuesField("without_index", 100 + i));
+ if (i % 17 == 0) {
+ iw.commit();
+ }
+ if (i != 55) {
+ doc.add(SortedNumericDocValuesField.indexedField("sparse", 100 + i));
+ }
+ if (i == 74) {
+ doc.add(SortedNumericDocValuesField.indexedField("super_sparse", 174));
+ }
+ iw.addDocument(doc);
+ }
+ iw.commit();
+
+ try (IndexReader reader = iw.getReader()) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ assertThat(
+ searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 0, 50)),
+ instanceOf(MatchNoDocsQuery.class));
+ assertThat(
+ searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 0, 250)),
+ instanceOf(MatchAllDocsQuery.class));
+ assertThat(
+ searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 50)),
+ instanceOf(MatchNoDocsQuery.class));
+ assertThat(
+ searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 0, 50)),
+ instanceOf(MatchNoDocsQuery.class));
+ assertThat(
+ searcher.rewrite(
+ SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 250, 350)),
+ instanceOf(MatchNoDocsQuery.class));
+ assertThat(
+ searcher
+ .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 174, 174))
+ .getClass()
+ .toString(),
+ containsString("SortedNumericDocValuesRangeQuery"));
+ assertThat(
+ searcher
+ .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 0, 150))
+ .getClass()
+ .toString(),
+ containsString("SortedNumericDocValuesRangeQuery"));
+ assertThat(
+ searcher
+ .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 150, 250))
+ .getClass()
+ .toString(),
+ containsString("SortedNumericDocValuesRangeQuery"));
+ assertThat(
+ searcher
+ .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 120, 150))
+ .getClass()
+ .toString(),
+ containsString("SortedNumericDocValuesRangeQuery"));
+ assertThat(
+ searcher
+ .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 250))
+ .getClass()
+ .toString(),
+ containsString("SortedNumericDocValuesRangeQuery"));
+ }
+ }
+ }
}