You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by zh...@apache.org on 2026/10/09 12:08:50 UTC
(lucene) branch main updated: Fix block-max support of DoubleValuesSource (#16697)
This is an automated email from the ASF dual-hosted git repository.
easyice pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new d150ba27d11 Fix block-max support of DoubleValuesSource (#16697)
d150ba27d11 is described below
commit d150ba27d1192396a8a67a765bb2f05cf79375b0
Author: Zhang Chao <80...@qq.com>
AuthorDate: Fri Oct 9 20:08:43 2026 +0800
Fix block-max support of DoubleValuesSource (#16697)
1. advanceShallow advanced the skipper on every call, including when the target was still inside the current block, which may cause the skipper to move to the next block.
2. getMaxScore,getMinScore only looked at level 0 of the skipper, including when upTo was beyond the current block, in which case the returned value only covers the current block, not the requested range.
---
lucene/CHANGES.txt | 5 +-
.../apache/lucene/search/DoubleValuesSource.java | 37 +++++++++---
.../lucene/search/TestDoubleValuesSource.java | 65 ++++++++++++++++++++++
3 files changed, 97 insertions(+), 10 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index fd562625fe5..448bf39d782 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -201,9 +201,8 @@ Improvements
Optimizations
---------------------
-
-* GITHUB#16431: Add support for Block-Max WAND dynamic score skipping in FunctionScoreQuery
- and DoubleValuesSource using DocValuesSkipper. (Rajat Jain)
+* GITHUB#16431, GITHUB#16697: Add support for Block-Max WAND dynamic score skipping in FunctionScoreQuery
+ and DoubleValuesSource using DocValuesSkipper. (Rajat Jain, Zhang Chao)
* GITHUB#16382: WildcardQuery.toAutomaton now collapses consecutive unescaped '*'
wildcards into a single any-string automaton, avoiding a quadratic blow-up in
diff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
index f7099484106..94e2bb6fc82 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
@@ -541,20 +541,42 @@ public abstract class DoubleValuesSource implements SegmentCacheable {
@Override
public int advanceShallow(int target) throws IOException {
if (skipper != null) {
- skipper.advance(target);
+ if (target > skipper.maxDocID(0)) {
+ skipper.advance(target);
+ }
return skipper.maxDocID(0);
}
return DocIdSetIterator.NO_MORE_DOCS;
}
+ /**
+ * Return the first level that includes all doc IDs up to {@code upTo}, or -1 if there is no
+ * such level.
+ */
+ private int getLevel(int upTo) {
+ final int maxDocID = skipper.maxDocID(0);
+ if (maxDocID == -1 || maxDocID == DocIdSetIterator.NO_MORE_DOCS) {
+ // the skipper's values may not be read before it is advanced or after it is exhausted
+ return -1;
+ }
+ for (int level = 0, numLevels = skipper.numLevels(); level < numLevels; ++level) {
+ final int skipperUpTo = skipper.maxDocID(level);
+ if (upTo <= skipperUpTo) {
+ return level;
+ }
+ }
+ return -1;
+ }
+
@Override
public float getMaxScore(int upTo) throws IOException {
if (skipper != null && monotonicity != Monotonicity.NONE) {
- if (skipper.minDocID(0) <= upTo) {
+ int level = getLevel(upTo);
+ if (level >= 0) {
long rawBound =
(monotonicity == Monotonicity.INCREASING)
- ? skipper.maxValue(0)
- : skipper.minValue(0);
+ ? skipper.maxValue(level)
+ : skipper.minValue(level);
return (float) decoder.applyAsDouble(rawBound);
}
}
@@ -564,11 +586,12 @@ public abstract class DoubleValuesSource implements SegmentCacheable {
@Override
public float getMinScore(int upTo) throws IOException {
if (skipper != null && monotonicity != Monotonicity.NONE) {
- if (skipper.minDocID(0) <= upTo) {
+ int level = getLevel(upTo);
+ if (level >= 0) {
long rawBound =
(monotonicity == Monotonicity.INCREASING)
- ? skipper.minValue(0)
- : skipper.maxValue(0);
+ ? skipper.minValue(level)
+ : skipper.maxValue(level);
return (float) decoder.applyAsDouble(rawBound);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
index b95162a2727..b041426549b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
@@ -21,15 +21,20 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatDocValuesField;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.AssertingDirectoryReader;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.search.CheckHits;
import org.apache.lucene.tests.util.English;
@@ -48,6 +53,9 @@ public class TestDoubleValuesSource extends LuceneTestCase {
private static IndexReader reader;
private static IndexSearcher searcher;
+ private static Directory skipIndexDir;
+ private static DirectoryReader skipIndexReader;
+
@BeforeClass
public static void beforeClass() throws Exception {
dir = newDirectory();
@@ -72,15 +80,22 @@ public class TestDoubleValuesSource extends LuceneTestCase {
reader = iw.getReader();
iw.close();
searcher = newSearcher(reader);
+
+ skipIndexDir = indexWithSkipIndex();
+ skipIndexReader = new AssertingDirectoryReader(DirectoryReader.open(skipIndexDir));
}
@AfterClass
public static void afterClass() throws Exception {
reader.close();
dir.close();
+ skipIndexReader.close();
+ skipIndexDir.close();
searcher = null;
reader = null;
dir = null;
+ skipIndexReader = null;
+ skipIndexDir = null;
}
public void testSortMissingZeroDefault() throws Exception {
@@ -365,4 +380,54 @@ public class TestDoubleValuesSource extends LuceneTestCase {
}
});
}
+
+ private static Directory indexWithSkipIndex() throws IOException {
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig();
+ conf.setCodec(TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(16)));
+ conf.setMergePolicy(newLogMergePolicy()); // we rely on docids following indexing order
+ try (IndexWriter writer = new IndexWriter(directory, conf)) {
+ for (int i = 0; i < 200; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field", i));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ }
+ return directory;
+ }
+
+ private static DoubleValues skipIndexValues() throws IOException {
+ LeafReaderContext context = skipIndexReader.leaves().get(0);
+ return DoubleValuesSource.fromLongField("field").getValues(context, null);
+ }
+
+ public void testAdvanceShallow() throws Exception {
+ DoubleValues values = skipIndexValues();
+ int blockEnd = values.advanceShallow(0);
+ assertEquals(blockEnd, values.advanceShallow(blockEnd));
+ assertTrue(values.advanceShallow(blockEnd + 1) > blockEnd);
+ }
+
+ public void testMaxScoreBeyondTheCurrentBlock() throws Exception {
+ DoubleValues values = skipIndexValues();
+ int upTo = values.advanceShallow(0) + 1;
+ assertTrue(values.advanceExact(upTo));
+ double score = values.doubleValue();
+ assertTrue(values.getMinScore(upTo) <= score);
+ assertTrue(score <= values.getMaxScore(upTo));
+
+ assertEquals(Float.POSITIVE_INFINITY, values.getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+ assertEquals(Float.NEGATIVE_INFINITY, values.getMinScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+ }
+
+ public void testMaxScoreAfterTheSkipperIsExhausted() throws Exception {
+ DoubleValues values = skipIndexValues();
+ values.advanceShallow(0);
+
+ assertEquals(
+ DocIdSetIterator.NO_MORE_DOCS, values.advanceShallow(DocIdSetIterator.NO_MORE_DOCS));
+ assertEquals(Float.POSITIVE_INFINITY, values.getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+ assertEquals(Float.NEGATIVE_INFINITY, values.getMinScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+ }
}