You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2025/06/03 05:30:18 UTC
(lucene) branch branch_10x updated (0386d68349d -> f2eb4d9fe29)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a change to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
from 0386d68349d Cache high-order bits of hashcode to speed up BytesRefHash (#14720)
new 92de7a20075 Keep evaluating conjunction one doc-at-a-time until dynamic pruning kicks in. (#14739)
new 5320141d040 Add MathUtil#unsignedMin to simplify dividing the doc ID space into windows. (#14750)
new 224c90127bc Add code comment about use of Collections#synchronizedMap in LRUQueryCache. (#14749)
new 42c79561ac6 Merge DocAndFreqBuffer and DocAndScoreBuffer. (#14748)
new f2eb4d9fe29 Merge CombinedFieldQuery's WeightedDisiWrapper into DisiWrapper. (#14747)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../codecs/lucene103/Lucene103PostingsReader.java | 8 ++-
.../java/org/apache/lucene/index/CheckIndex.java | 8 +--
.../java/org/apache/lucene/index/PostingsEnum.java | 10 ++--
.../apache/lucene/search/BatchScoreBulkScorer.java | 4 +-
.../apache/lucene/search/BitSetDocIdStream.java | 3 +-
.../search/BlockMaxConjunctionBulkScorer.java | 68 ++++++++++++++++++----
.../org/apache/lucene/search/BooleanScorer.java | 4 +-
.../apache/lucene/search/CombinedFieldQuery.java | 25 ++------
.../lucene/search/DenseConjunctionBulkScorer.java | 5 +-
.../java/org/apache/lucene/search/DisiWrapper.java | 13 +++++
.../lucene/search/DisjunctionMaxBulkScorer.java | 3 +-
...eqBuffer.java => DocAndFloatFeatureBuffer.java} | 19 +++---
.../apache/lucene/search/DocAndScoreAccBuffer.java | 9 ++-
.../apache/lucene/search/DocAndScoreBuffer.java | 50 ----------------
.../org/apache/lucene/search/LRUQueryCache.java | 3 +
.../apache/lucene/search/MaxScoreBulkScorer.java | 12 ++--
.../src/java/org/apache/lucene/search/Scorer.java | 8 +--
.../java/org/apache/lucene/search/TermScorer.java | 24 +++-----
.../lucene/search/TimeLimitingBulkScorer.java | 3 +-
.../org/apache/lucene/util/BitSetIterator.java | 2 +-
.../apache/lucene/util/DocBaseBitSetIterator.java | 2 +-
.../src/java/org/apache/lucene/util/MathUtil.java | 5 ++
.../org/apache/lucene/search/TestTermScorer.java | 4 +-
.../test/org/apache/lucene/util/TestMathUtil.java | 15 +++++
.../lucene/tests/index/AssertingLeafReader.java | 4 +-
.../lucene/tests/index/RandomPostingsTester.java | 6 +-
.../lucene/tests/search/AssertingScorer.java | 4 +-
27 files changed, 171 insertions(+), 150 deletions(-)
rename lucene/core/src/java/org/apache/lucene/search/{DocAndFreqBuffer.java => DocAndFloatFeatureBuffer.java} (77%)
delete mode 100644 lucene/core/src/java/org/apache/lucene/search/DocAndScoreBuffer.java
(lucene) 03/05: Add code comment about use of Collections#synchronizedMap in LRUQueryCache. (#14749)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 224c90127bc9f62a90c6a3ac884cdadc0692c971
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Jun 3 07:05:46 2025 +0200
Add code comment about use of Collections#synchronizedMap in LRUQueryCache. (#14749)
It may not be totally obvious why this map needs to be synchronized since reads
and writes are performed under a read-write lock. Except that unlike most other
collections, reads are not thread-safe on a `LinkedHashMap`.
Closes #14677
Co-authored-by: Benjamin Trent <be...@gmail.com>
---
lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
index 088a47bf0a9..05ef52ff327 100644
--- a/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
+++ b/lucene/core/src/java/org/apache/lucene/search/LRUQueryCache.java
@@ -131,6 +131,9 @@ public class LRUQueryCache implements QueryCache, Accountable {
}
this.skipCacheFactor = skipCacheFactor;
+ // Note that reads on this LinkedHashMap trigger modifications on the linked list under the
+ // hood, so reading from multiple threads is not thread-safe. This is why it is wrapped in a
+ // Collections#synchronizedMap.
uniqueQueries = Collections.synchronizedMap(new LinkedHashMap<>(16, 0.75f, true));
mostRecentlyUsedQueries = uniqueQueries.keySet();
cache = new IdentityHashMap<>();
(lucene) 02/05: Add MathUtil#unsignedMin to simplify dividing the doc ID space into windows. (#14750)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 5320141d040ad3006df958714819b8c830c23292
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Jun 3 07:05:26 2025 +0200
Add MathUtil#unsignedMin to simplify dividing the doc ID space into windows. (#14750)
We often need to divide the doc ID space into sub windows, but this is prone to
integer overflows if the size of the window is greater than `Integer.MAX_VALUE
- windowStart`. This adds `MathUtil#unsignedMin` to make these overflows a bit
easier to manage.
---
.../java/org/apache/lucene/search/BitSetDocIdStream.java | 3 ++-
.../apache/lucene/search/DenseConjunctionBulkScorer.java | 5 +++--
.../apache/lucene/search/DisjunctionMaxBulkScorer.java | 3 ++-
.../java/org/apache/lucene/search/MaxScoreBulkScorer.java | 8 ++++----
.../org/apache/lucene/search/TimeLimitingBulkScorer.java | 3 ++-
.../src/java/org/apache/lucene/util/BitSetIterator.java | 2 +-
.../org/apache/lucene/util/DocBaseBitSetIterator.java | 2 +-
lucene/core/src/java/org/apache/lucene/util/MathUtil.java | 5 +++++
.../src/test/org/apache/lucene/util/TestMathUtil.java | 15 +++++++++++++++
9 files changed, 35 insertions(+), 11 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/search/BitSetDocIdStream.java b/lucene/core/src/java/org/apache/lucene/search/BitSetDocIdStream.java
index 42a75073c81..8c2f01e38e3 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BitSetDocIdStream.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BitSetDocIdStream.java
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.MathUtil;
final class BitSetDocIdStream extends DocIdStream {
@@ -29,7 +30,7 @@ final class BitSetDocIdStream extends DocIdStream {
this.bitSet = bitSet;
this.offset = offset;
upTo = offset;
- max = (int) Math.min(Integer.MAX_VALUE, (long) offset + bitSet.length());
+ max = MathUtil.unsignedMin(Integer.MAX_VALUE, offset + bitSet.length());
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java
index 22909d45400..08c71a81d7d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java
@@ -22,6 +22,7 @@ import java.util.Comparator;
import java.util.List;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.MathUtil;
/**
* BulkScorer implementation of {@link ConjunctionScorer} that is specialized for dense clauses.
@@ -176,7 +177,7 @@ final class DenseConjunctionBulkScorer extends BulkScorer {
// data, which helps evaluate fewer clauses per window - without allowing windows to become too
// small thanks to the WINDOW_SIZE/2 threshold.
int minDocIDRunEnd = max;
- final int minRunEndThreshold = (int) Math.min((long) min + WINDOW_SIZE / 2, max);
+ final int minRunEndThreshold = MathUtil.unsignedMin(min + WINDOW_SIZE / 2, max);
for (DisiWrapper w : iterators) {
int docIdRunEnd = w.docIDRunEnd();
if (w.docID() > min || docIdRunEnd < minRunEndThreshold) {
@@ -195,7 +196,7 @@ final class DenseConjunctionBulkScorer extends BulkScorer {
return minDocIDRunEnd;
}
- int bitsetWindowMax = (int) Math.min(minDocIDRunEnd, (long) WINDOW_SIZE + min);
+ int bitsetWindowMax = MathUtil.unsignedMin(minDocIDRunEnd, WINDOW_SIZE + min);
if (windowTwoPhases.isEmpty()) {
scoreWindowUsingBitSet(collector, acceptDocs, windowApproximations, min, bitsetWindowMax);
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxBulkScorer.java
index 1ee78c3e817..78184b48eda 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxBulkScorer.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Objects;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.MathUtil;
import org.apache.lucene.util.PriorityQueue;
/** Bulk scorer for {@link DisjunctionMaxQuery} when the tie-break multiplier is zero. */
@@ -67,7 +68,7 @@ final class DisjunctionMaxBulkScorer extends BulkScorer {
while (top.next < max) {
final int windowMin = Math.max(top.next, min);
- final int windowMax = (int) Math.min(max, (long) windowMin + WINDOW_SIZE);
+ final int windowMax = MathUtil.unsignedMin(max, windowMin + WINDOW_SIZE);
// First compute matches / scores in the window
do {
diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
index cb212096f75..f6e83c7af52 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
@@ -181,7 +181,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
// Only score an inner window, after that we'll check if the min competitive score has increased
// enough for a more favorable partitioning to be used.
int innerWindowMin = top.doc;
- int innerWindowMax = (int) Math.min(max, (long) innerWindowMin + INNER_WINDOW_SIZE);
+ int innerWindowMax = MathUtil.unsignedMin(max, innerWindowMin + INNER_WINDOW_SIZE);
docAndScoreAccBuffer.size = 0;
while (top.doc < innerWindowMax) {
@@ -277,7 +277,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
DisiWrapper top = essentialQueue.top();
int innerWindowMin = top.doc;
- int innerWindowMax = (int) Math.min(max, (long) innerWindowMin + INNER_WINDOW_SIZE);
+ int innerWindowMax = MathUtil.unsignedMin(max, innerWindowMin + INNER_WINDOW_SIZE);
int innerWindowSize = innerWindowMax - innerWindowMin;
// Collect matches of essential clauses into a bitset
@@ -324,7 +324,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
final DisiWrapper scorer = allScorers[i];
if (filter == null || scorer.cost >= filter.cost) {
final int upTo = scorer.scorer.advanceShallow(Math.max(scorer.doc, windowMin));
- windowMax = (int) Math.min(windowMax, upTo + 1L); // upTo is inclusive
+ windowMax = MathUtil.unsignedMin(windowMax, upTo + 1); // upTo is inclusive
}
}
@@ -341,7 +341,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
minWindowSize = 1;
}
- int minWindowMax = (int) Math.min(Integer.MAX_VALUE, (long) windowMin + minWindowSize);
+ int minWindowMax = MathUtil.unsignedMin(Integer.MAX_VALUE, windowMin + minWindowSize);
windowMax = Math.max(windowMax, minWindowMax);
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/TimeLimitingBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/TimeLimitingBulkScorer.java
index 240dee7f4fd..63e6ba1b294 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TimeLimitingBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TimeLimitingBulkScorer.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.index.QueryTimeout;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.MathUtil;
/**
* The {@link TimeLimitingBulkScorer} is used to timeout search requests that take longer than the
@@ -69,7 +70,7 @@ final class TimeLimitingBulkScorer extends BulkScorer {
public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
int interval = INTERVAL;
while (min < max) {
- final int newMax = (int) Math.min((long) min + interval, max);
+ final int newMax = MathUtil.unsignedMin(min + interval, max);
final int newInterval =
interval + (interval >> 1); // increase the interval by 50% on each iteration
// overflow check
diff --git a/lucene/core/src/java/org/apache/lucene/util/BitSetIterator.java b/lucene/core/src/java/org/apache/lucene/util/BitSetIterator.java
index 23938d9a59b..11ab25700f4 100644
--- a/lucene/core/src/java/org/apache/lucene/util/BitSetIterator.java
+++ b/lucene/core/src/java/org/apache/lucene/util/BitSetIterator.java
@@ -100,7 +100,7 @@ public class BitSetIterator extends AbstractDocIdSetIterator {
// The destination bit set may be shorter than this bit set. This is only legal if all bits
// beyond offset + bitSet.length() are clear. If not, the below call to `super.intoBitSet`
// will throw an exception.
- actualUpto = (int) Math.min(actualUpto, offset + (long) bitSet.length());
+ actualUpto = MathUtil.unsignedMin(actualUpto, offset + bitSet.length());
FixedBitSet.orRange(fixedBits, doc, bitSet, doc - offset, actualUpto - doc);
advance(actualUpto); // set the current doc
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/DocBaseBitSetIterator.java b/lucene/core/src/java/org/apache/lucene/util/DocBaseBitSetIterator.java
index 14a90227168..a7f083b1bda 100644
--- a/lucene/core/src/java/org/apache/lucene/util/DocBaseBitSetIterator.java
+++ b/lucene/core/src/java/org/apache/lucene/util/DocBaseBitSetIterator.java
@@ -92,7 +92,7 @@ public class DocBaseBitSetIterator extends AbstractDocIdSetIterator {
// The destination bit set may be shorter than this bit set. This is only legal if all bits
// beyond offset + bitSet.length() are clear. If not, the below call to `super.intoBitSet` will
// throw an exception.
- actualUpto = (int) Math.min(actualUpto, offset + (long) bitSet.length());
+ actualUpto = MathUtil.unsignedMin(actualUpto, offset + bitSet.length());
if (actualUpto > doc) {
FixedBitSet.orRange(bits, doc - docBase, bitSet, doc - offset, actualUpto - doc);
advance(actualUpto); // set the current doc
diff --git a/lucene/core/src/java/org/apache/lucene/util/MathUtil.java b/lucene/core/src/java/org/apache/lucene/util/MathUtil.java
index 141a4566595..aecf48adaf6 100644
--- a/lucene/core/src/java/org/apache/lucene/util/MathUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/util/MathUtil.java
@@ -192,4 +192,9 @@ public final class MathUtil {
double b = MathUtil.sumRelativeErrorBound(numValues);
return (1.0 + 2 * b) * sum;
}
+
+ /** Return the min of the two given unsigned integers. */
+ public static int unsignedMin(int a, int b) {
+ return Integer.compareUnsigned(a, b) < 0 ? a : b;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java b/lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java
index e36df8c51cd..c44e52f6cbd 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java
@@ -181,4 +181,19 @@ public class TestMathUtil extends LuceneTestCase {
assertEquals(0.5493061443340549, MathUtil.atanh(0.5), epsilon);
assertEquals(Double.POSITIVE_INFINITY, MathUtil.atanh(1), 0);
}
+
+ public void testUnsignedMin() {
+ assertEquals(0, MathUtil.unsignedMin(0, 0));
+ assertEquals(0, MathUtil.unsignedMin(0, 3));
+ assertEquals(0, MathUtil.unsignedMin(3, 0));
+ assertEquals(0, MathUtil.unsignedMin(0, Integer.MAX_VALUE));
+ assertEquals(0, MathUtil.unsignedMin(Integer.MAX_VALUE, 0));
+ assertEquals(Integer.MAX_VALUE, MathUtil.unsignedMin(Integer.MAX_VALUE, Integer.MAX_VALUE + 1));
+ assertEquals(Integer.MAX_VALUE, MathUtil.unsignedMin(Integer.MAX_VALUE + 1, Integer.MAX_VALUE));
+ assertEquals(Integer.MAX_VALUE, MathUtil.unsignedMin(Integer.MAX_VALUE, Integer.MIN_VALUE));
+ assertEquals(Integer.MAX_VALUE, MathUtil.unsignedMin(Integer.MIN_VALUE, Integer.MAX_VALUE));
+ assertEquals(Integer.MIN_VALUE, MathUtil.unsignedMin(Integer.MIN_VALUE, -1));
+ assertEquals(Integer.MIN_VALUE, MathUtil.unsignedMin(-1, Integer.MIN_VALUE));
+ assertEquals(Integer.MIN_VALUE, MathUtil.unsignedMin(Integer.MIN_VALUE, Integer.MIN_VALUE));
+ }
}
(lucene) 04/05: Merge DocAndFreqBuffer and DocAndScoreBuffer. (#14748)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 42c79561ac6c4ef565a51d6d196bf29d4ea266ce
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Jun 3 07:06:21 2025 +0200
Merge DocAndFreqBuffer and DocAndScoreBuffer. (#14748)
I initially kept those separated but I'm now coming to the conclusion that they
should be merged:
- While frequencies are ints today, some queries produce float frequencies
(`SloppyPhraseQuery`, `CombinedFieldQuery`).
- `TermScorer` no longer needs to keep track of a separate buffer for
frequencies.
Co-authored-by: Alan Woodward <ro...@gmail.com>
---
.../codecs/lucene103/Lucene103PostingsReader.java | 8 ++--
.../java/org/apache/lucene/index/CheckIndex.java | 8 ++--
.../java/org/apache/lucene/index/PostingsEnum.java | 10 ++---
.../apache/lucene/search/BatchScoreBulkScorer.java | 4 +-
.../search/BlockMaxConjunctionBulkScorer.java | 2 +-
.../org/apache/lucene/search/BooleanScorer.java | 4 +-
...eqBuffer.java => DocAndFloatFeatureBuffer.java} | 19 ++++----
.../apache/lucene/search/DocAndScoreAccBuffer.java | 9 ++--
.../apache/lucene/search/DocAndScoreBuffer.java | 50 ----------------------
.../apache/lucene/search/MaxScoreBulkScorer.java | 4 +-
.../src/java/org/apache/lucene/search/Scorer.java | 8 ++--
.../java/org/apache/lucene/search/TermScorer.java | 24 ++++-------
.../org/apache/lucene/search/TestTermScorer.java | 4 +-
.../lucene/tests/index/AssertingLeafReader.java | 4 +-
.../lucene/tests/index/RandomPostingsTester.java | 6 +--
.../lucene/tests/search/AssertingScorer.java | 4 +-
16 files changed, 59 insertions(+), 109 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
index afc523a3896..cab3c3cc4bc 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
@@ -46,7 +46,7 @@ import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.internal.vectorization.VectorizationProvider;
-import org.apache.lucene.search.DocAndFreqBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
@@ -1043,7 +1043,7 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {
}
@Override
- public void nextPostings(int upTo, DocAndFreqBuffer buffer) throws IOException {
+ public void nextPostings(int upTo, DocAndFloatFeatureBuffer buffer) throws IOException {
assert needsRefilling == false;
if (needsFreq == false) {
@@ -1080,7 +1080,9 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {
}
assert buffer.size > 0;
- System.arraycopy(freqBuffer, start, buffer.freqs, 0, buffer.size);
+ for (int i = 0; i < buffer.size; ++i) {
+ buffer.features[i] = freqBuffer[start + i];
+ }
advance(upTo);
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index b6f38c2860d..ad8fab75c98 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -58,7 +58,7 @@ import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.search.DocAndFreqBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldExistsQuery;
import org.apache.lucene.search.KnnCollector;
@@ -1614,7 +1614,7 @@ public final class CheckIndex implements Closeable {
postings = termsEnum.postings(postings, PostingsEnum.ALL);
bulkPostings = termsEnum.postings(bulkPostings, PostingsEnum.ALL);
bulkPostings.nextDoc();
- DocAndFreqBuffer buffer = new DocAndFreqBuffer();
+ DocAndFloatFeatureBuffer buffer = new DocAndFloatFeatureBuffer();
int bufferIndex = 0;
if (hasFreqs == false) {
@@ -1680,10 +1680,10 @@ public final class CheckIndex implements Closeable {
+ " as next doc while PostingsEnum#nextDoc returns "
+ doc);
}
- if (freq != buffer.freqs[bufferIndex]) {
+ if (freq != buffer.features[bufferIndex]) {
throw new CheckIndexException(
"PostingsEnum#nextPostings returns "
- + buffer.freqs[bufferIndex]
+ + buffer.features[bufferIndex]
+ " as term freq while PostingsEnum#freq returns "
+ freq);
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java b/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java
index 8e82e0a6d69..8fd9d8c9a37 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java
@@ -17,7 +17,7 @@
package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.search.DocAndFreqBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
@@ -126,18 +126,18 @@ public abstract class PostingsEnum extends DocIdSetIterator {
* buffer.size = size;
* </pre>
*
- * <p><b>NOTE</b>: The provided {@link DocAndFreqBuffer} should not hold references to internal
- * data structures.
+ * <p><b>NOTE</b>: The provided {@link DocAndFloatFeatureBuffer} should not hold references to
+ * internal data structures.
*
* @lucene.internal
*/
- public void nextPostings(int upTo, DocAndFreqBuffer buffer) throws IOException {
+ public void nextPostings(int upTo, DocAndFloatFeatureBuffer buffer) throws IOException {
int batchSize = 16; // arbitrary
buffer.growNoCopy(batchSize);
int size = 0;
for (int doc = docID(); doc < upTo && size < batchSize; doc = nextDoc()) {
buffer.docs[size] = doc;
- buffer.freqs[size] = freq();
+ buffer.features[size] = freq();
++size;
}
buffer.size = size;
diff --git a/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java
index c000209d2fc..d1412fb5e45 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java
@@ -26,7 +26,7 @@ import org.apache.lucene.util.Bits;
class BatchScoreBulkScorer extends BulkScorer {
private final SimpleScorable scorable = new SimpleScorable();
- private final DocAndScoreBuffer buffer = new DocAndScoreBuffer();
+ private final DocAndFloatFeatureBuffer buffer = new DocAndFloatFeatureBuffer();
private final Scorer scorer;
BatchScoreBulkScorer(Scorer scorer) {
@@ -50,7 +50,7 @@ class BatchScoreBulkScorer extends BulkScorer {
buffer.size > 0;
scorer.nextDocsAndScores(max, acceptDocs, buffer)) {
for (int i = 0, size = buffer.size; i < size; i++) {
- float score = scorable.score = buffer.scores[i];
+ float score = scorable.score = buffer.features[i];
if (score >= scorable.minCompetitiveScore) {
collector.collect(buffer.docs[i]);
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java
index e2e063feb5e..31904eeff7e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java
@@ -41,7 +41,7 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {
private final DocAndScore scorable = new DocAndScore();
private final double[] sumOfOtherClauses;
private final int maxDoc;
- private final DocAndScoreBuffer docAndScoreBuffer = new DocAndScoreBuffer();
+ private final DocAndFloatFeatureBuffer docAndScoreBuffer = new DocAndFloatFeatureBuffer();
private final DocAndScoreAccBuffer docAndScoreAccBuffer = new DocAndScoreAccBuffer();
BlockMaxConjunctionBulkScorer(int maxDoc, List<Scorer> scorers) throws IOException {
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
index 1f4dd4f80ce..9bb2ec7882a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
@@ -81,7 +81,7 @@ final class BooleanScorer extends BulkScorer {
final int minShouldMatch;
final long cost;
final boolean needsScores;
- private final DocAndScoreBuffer docAndScoreBuffer = new DocAndScoreBuffer();
+ private final DocAndFloatFeatureBuffer docAndScoreBuffer = new DocAndFloatFeatureBuffer();
BooleanScorer(Collection<Scorer> scorers, int minShouldMatch, boolean needsScores) {
if (minShouldMatch < 1 || minShouldMatch > scorers.size()) {
@@ -148,7 +148,7 @@ final class BooleanScorer extends BulkScorer {
w.scorer.nextDocsAndScores(max, acceptDocs, docAndScoreBuffer)) {
for (int index = 0; index < docAndScoreBuffer.size; ++index) {
final int doc = docAndScoreBuffer.docs[index];
- final float score = docAndScoreBuffer.scores[index];
+ final float score = docAndScoreBuffer.features[index];
final int d = doc & MASK;
matching.set(d);
final Bucket bucket = buckets[d];
diff --git a/lucene/core/src/java/org/apache/lucene/search/DocAndFreqBuffer.java b/lucene/core/src/java/org/apache/lucene/search/DocAndFloatFeatureBuffer.java
similarity index 77%
rename from lucene/core/src/java/org/apache/lucene/search/DocAndFreqBuffer.java
rename to lucene/core/src/java/org/apache/lucene/search/DocAndFloatFeatureBuffer.java
index 3e45d3cf429..78fd7dd1055 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DocAndFreqBuffer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DocAndFloatFeatureBuffer.java
@@ -21,29 +21,32 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IntsRef;
/**
- * Wrapper around parallel arrays storing doc IDs and their corresponding frequencies.
+ * Wrapper around parallel arrays storing doc IDs and their corresponding features, stored as Java
+ * floats. These features may be anything, but are typically a term frequency or a score.
*
* @lucene.internal
*/
-public final class DocAndFreqBuffer {
+public final class DocAndFloatFeatureBuffer {
+
+ private static final float[] EMPTY_FLOATS = new float[0];
/** Doc IDs */
public int[] docs = IntsRef.EMPTY_INTS;
- /** Frequencies */
- public int[] freqs = IntsRef.EMPTY_INTS;
+ /** Float-valued features */
+ public float[] features = EMPTY_FLOATS;
- /** Number of valid entries in the doc ID and frequency arrays. */
+ /** Number of valid entries in the doc ID and float-valued feature arrays. */
public int size;
/** Sole constructor. */
- public DocAndFreqBuffer() {}
+ public DocAndFloatFeatureBuffer() {}
/** Grow both arrays to ensure that they can store at least the given number of entries. */
public void growNoCopy(int minSize) {
if (docs.length < minSize) {
docs = ArrayUtil.growNoCopy(docs, minSize);
- freqs = new int[docs.length];
+ features = new float[docs.length];
}
}
@@ -53,7 +56,7 @@ public final class DocAndFreqBuffer {
for (int i = 0; i < size; ++i) {
if (liveDocs.get(docs[i])) {
docs[newSize] = docs[i];
- freqs[newSize] = freqs[i];
+ features[newSize] = features[i];
newSize++;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/DocAndScoreAccBuffer.java b/lucene/core/src/java/org/apache/lucene/search/DocAndScoreAccBuffer.java
index dd05e53aac2..ed5ffbc3f9a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DocAndScoreAccBuffer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DocAndScoreAccBuffer.java
@@ -62,12 +62,15 @@ public final class DocAndScoreAccBuffer {
}
}
- /** Copy content from the given {@link DocAndScoreBuffer}, expanding float scores to doubles. */
- public void copyFrom(DocAndScoreBuffer buffer) {
+ /**
+ * Copy content from the given {@link DocAndFloatFeatureBuffer}, expanding float scores to
+ * doubles.
+ */
+ public void copyFrom(DocAndFloatFeatureBuffer buffer) {
growNoCopy(buffer.size);
System.arraycopy(buffer.docs, 0, docs, 0, buffer.size);
for (int i = 0; i < buffer.size; ++i) {
- scores[i] = buffer.scores[i];
+ scores[i] = buffer.features[i];
}
this.size = buffer.size;
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/DocAndScoreBuffer.java b/lucene/core/src/java/org/apache/lucene/search/DocAndScoreBuffer.java
deleted file mode 100644
index 5daef38c0ff..00000000000
--- a/lucene/core/src/java/org/apache/lucene/search/DocAndScoreBuffer.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search;
-
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.IntsRef;
-
-/**
- * Wrapper around parallel arrays storing doc IDs and their corresponding scores.
- *
- * @lucene.internal
- */
-public final class DocAndScoreBuffer {
-
- private static final float[] EMPTY_FLOATS = new float[0];
-
- /** Doc IDs */
- public int[] docs = IntsRef.EMPTY_INTS;
-
- /** Scores */
- public float[] scores = EMPTY_FLOATS;
-
- /** Number of valid entries in the doc ID and score arrays. */
- public int size;
-
- /** Sole constructor. */
- public DocAndScoreBuffer() {}
-
- /** Grow both arrays to ensure that they can store at least the given number of entries. */
- public void growNoCopy(int minSize) {
- if (docs.length < minSize) {
- docs = ArrayUtil.growNoCopy(docs, minSize);
- scores = new float[docs.length];
- }
- }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
index f6e83c7af52..87135f91147 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
@@ -51,7 +51,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
private final FixedBitSet windowMatches = new FixedBitSet(INNER_WINDOW_SIZE);
private final double[] windowScores = new double[INNER_WINDOW_SIZE];
- private final DocAndScoreBuffer docAndScoreBuffer = new DocAndScoreBuffer();
+ private final DocAndFloatFeatureBuffer docAndScoreBuffer = new DocAndFloatFeatureBuffer();
private final DocAndScoreAccBuffer docAndScoreAccBuffer = new DocAndScoreAccBuffer();
MaxScoreBulkScorer(int maxDoc, List<Scorer> scorers, Scorer filter) throws IOException {
@@ -287,7 +287,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
top.scorer.nextDocsAndScores(innerWindowMax, acceptDocs, docAndScoreBuffer)) {
for (int index = 0; index < docAndScoreBuffer.size; ++index) {
final int doc = docAndScoreBuffer.docs[index];
- final float score = docAndScoreBuffer.scores[index];
+ final float score = docAndScoreBuffer.features[index];
final int i = doc - innerWindowMin;
windowMatches.set(i);
windowScores[i] += score;
diff --git a/lucene/core/src/java/org/apache/lucene/search/Scorer.java b/lucene/core/src/java/org/apache/lucene/search/Scorer.java
index 64db31ba9fd..86207f5c18c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Scorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Scorer.java
@@ -107,15 +107,15 @@ public abstract class Scorer extends Scorable {
* buffer.size = size;
* </pre>
*
- * <p><b>NOTE</b>: The provided {@link DocAndScoreBuffer} should not hold references to internal
- * data structures.
+ * <p><b>NOTE</b>: The provided {@link DocAndFloatFeatureBuffer} should not hold references to
+ * internal data structures.
*
* <p><b>NOTE</b>: In case this {@link Scorer} exposes a {@link #twoPhaseIterator()
* TwoPhaseIterator}, it should be positioned on a matching document before this method is called.
*
* @lucene.internal
*/
- public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndScoreBuffer buffer)
+ public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndFloatFeatureBuffer buffer)
throws IOException {
int batchSize = 16; // arbitrary
buffer.growNoCopy(batchSize);
@@ -124,7 +124,7 @@ public abstract class Scorer extends Scorable {
for (int doc = docID(); doc < upTo && size < batchSize; doc = iterator.nextDoc()) {
if (liveDocs == null || liveDocs.get(doc)) {
buffer.docs[size] = doc;
- buffer.scores[size] = score();
+ buffer.features[size] = score();
++size;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
index db782692f45..278f735838e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
@@ -39,7 +39,6 @@ public final class TermScorer extends Scorer {
private final NumericDocValues norms;
private final ImpactsDISI impactsDisi;
private final MaxScoreCache maxScoreCache;
- private DocAndFreqBuffer docAndFreqBuffer;
private long[] normValues = LongsRef.EMPTY_LONGS;
/** Construct a {@link TermScorer} that will iterate all documents. */
@@ -128,32 +127,28 @@ public final class TermScorer extends Scorer {
}
@Override
- public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndScoreBuffer buffer)
+ public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndFloatFeatureBuffer buffer)
throws IOException {
- if (docAndFreqBuffer == null) {
- docAndFreqBuffer = new DocAndFreqBuffer();
- }
-
for (; ; ) {
if (impactsDisi != null) {
impactsDisi.ensureCompetitive();
}
- postingsEnum.nextPostings(upTo, docAndFreqBuffer);
- if (liveDocs != null && docAndFreqBuffer.size != 0) {
+ postingsEnum.nextPostings(upTo, buffer);
+ if (liveDocs != null && buffer.size != 0) {
// An empty return value indicates that there are no more docs before upTo. We may be
// unlucky, and there are docs left, but all docs from the current batch happen to be marked
// as deleted. So we need to iterate until we find a batch that has at least one non-deleted
// doc.
- docAndFreqBuffer.apply(liveDocs);
- if (docAndFreqBuffer.size == 0) {
+ buffer.apply(liveDocs);
+ if (buffer.size == 0) {
continue;
}
}
break;
}
- int size = docAndFreqBuffer.size;
+ int size = buffer.size;
if (normValues.length < size) {
normValues = new long[ArrayUtil.oversize(size, Long.BYTES)];
if (norms == null) {
@@ -162,7 +157,7 @@ public final class TermScorer extends Scorer {
}
if (norms != null) {
for (int i = 0; i < size; ++i) {
- if (norms.advanceExact(docAndFreqBuffer.docs[i])) {
+ if (norms.advanceExact(buffer.docs[i])) {
normValues[i] = norms.longValue();
} else {
normValues[i] = 1L;
@@ -170,13 +165,10 @@ public final class TermScorer extends Scorer {
}
}
- buffer.growNoCopy(size);
- buffer.size = size;
- System.arraycopy(docAndFreqBuffer.docs, 0, buffer.docs, 0, size);
for (int i = 0; i < size; ++i) {
// Unless SimScorer#score is megamorphic, SimScorer#score should inline and (part of) score
// computations should auto-vectorize.
- buffer.scores[i] = scorer.score(docAndFreqBuffer.freqs[i], normValues[i]);
+ buffer.features[i] = scorer.score(buffer.features[i], normValues[i]);
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
index 3264af9dc8b..c2dfa2e6e6a 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
@@ -281,7 +281,7 @@ public class TestTermScorer extends LuceneTestCase {
Scorer scorer2 = weight.scorer(context);
scorer1.iterator().nextDoc();
scorer2.iterator().nextDoc();
- DocAndScoreBuffer buffer = new DocAndScoreBuffer();
+ DocAndFloatFeatureBuffer buffer = new DocAndFloatFeatureBuffer();
while (true) {
int curDoc = scorer2.iterator().docID();
int upTo =
@@ -294,7 +294,7 @@ public class TestTermScorer extends LuceneTestCase {
scorer2.iterator().nextDoc();
}
assertEquals(scorer2.iterator().docID(), buffer.docs[i]);
- assertEquals(scorer2.score(), buffer.scores[i], 0f);
+ assertEquals(scorer2.score(), buffer.features[i], 0f);
scorer2.iterator().nextDoc();
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java
index 898d93fdc0f..256e13c514e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java
@@ -50,7 +50,7 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.internal.tests.IndexPackageAccess;
import org.apache.lucene.internal.tests.TestSecrets;
-import org.apache.lucene.search.DocAndFreqBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -598,7 +598,7 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
- public void nextPostings(int upTo, DocAndFreqBuffer buffer) throws IOException {
+ public void nextPostings(int upTo, DocAndFloatFeatureBuffer buffer) throws IOException {
assert state != DocsEnumState.START : "nextPostings() called before nextDoc()/advance()";
in.nextPostings(upTo, buffer);
doc = in.docID();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java
index 9613e1ba6c2..47dc3ae2f6c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java
@@ -66,7 +66,7 @@ import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.internal.tests.IndexPackageAccess;
import org.apache.lucene.internal.tests.TestSecrets;
-import org.apache.lucene.search.DocAndFreqBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
@@ -1443,7 +1443,7 @@ public class RandomPostingsTester {
pe1.nextDoc();
pe2.nextDoc();
- DocAndFreqBuffer buffer = new DocAndFreqBuffer();
+ DocAndFloatFeatureBuffer buffer = new DocAndFloatFeatureBuffer();
while (true) {
int curDoc = pe1.docID();
int upTo =
@@ -1454,7 +1454,7 @@ public class RandomPostingsTester {
for (int i = 0; i < buffer.size; ++i) {
assertTrue(buffer.docs[i] < upTo);
assertEquals(pe2.docID(), buffer.docs[i]);
- assertEquals(pe2.freq(), buffer.freqs[i]);
+ assertEquals(0f, pe2.freq(), buffer.features[i]);
pe2.nextDoc();
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/AssertingScorer.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/AssertingScorer.java
index 1910a48d074..f0ef919761c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/AssertingScorer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/AssertingScorer.java
@@ -19,7 +19,7 @@ package org.apache.lucene.tests.search;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
-import org.apache.lucene.search.DocAndScoreBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FilterDocIdSetIterator;
import org.apache.lucene.search.Scorer;
@@ -286,7 +286,7 @@ public class AssertingScorer extends Scorer {
}
@Override
- public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndScoreBuffer buffer)
+ public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndFloatFeatureBuffer buffer)
throws IOException {
assert doc != -1;
in.nextDocsAndScores(upTo, liveDocs, buffer);
(lucene) 01/05: Keep evaluating conjunction one doc-at-a-time until dynamic pruning kicks in. (#14739)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 92de7a200751132e3470284c78793ce2f9aa477c
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Mon Jun 2 17:03:05 2025 +0200
Keep evaluating conjunction one doc-at-a-time until dynamic pruning kicks in. (#14739)
This essentially reverts the change from #14701 for conjunctive queries that
have not reached their `totalHitsThreshold` yet. This should speed up queries
whose total number of matches is in the order of `totalHitsThreshold` or less,
such as filtered conjunctions on nightly benchmarks.
---
.../search/BlockMaxConjunctionBulkScorer.java | 66 ++++++++++++++++++----
1 file changed, 55 insertions(+), 11 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java
index 7091777a262..e2e063feb5e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java
@@ -87,18 +87,64 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {
// NOTE: windowMax is inclusive
int windowMax = Math.min(scorers[0].advanceShallow(windowMin), max - 1);
- float maxWindowScore = Float.POSITIVE_INFINITY;
if (0 < scorable.minCompetitiveScore) {
- maxWindowScore = computeMaxScore(windowMin, windowMax);
+ float maxWindowScore = computeMaxScore(windowMin, windowMax);
+ scoreWindowScoreFirst(collector, acceptDocs, windowMin, windowMax + 1, maxWindowScore);
+ } else {
+ scoreWindowDocFirst(collector, acceptDocs, windowMin, windowMax + 1);
}
- scoreWindow(collector, acceptDocs, windowMin, windowMax + 1, maxWindowScore);
windowMin = Math.max(lead.docID(), windowMax + 1);
}
return windowMin >= maxDoc ? DocIdSetIterator.NO_MORE_DOCS : windowMin;
}
- private void scoreWindow(
+ /**
+ * Score a window of doc IDs by first finding agreement between all iterators, and only then
+ * compute scores and call the collector.
+ */
+ private void scoreWindowDocFirst(LeafCollector collector, Bits acceptDocs, int min, int max)
+ throws IOException {
+ int doc = lead.docID();
+ if (doc < min) {
+ doc = lead.advance(min);
+ }
+
+ outer:
+ while (doc < max) {
+ if (acceptDocs == null || acceptDocs.get(doc)) {
+ for (int i = 1; i < iterators.length; ++i) {
+ DocIdSetIterator iterator = iterators[i];
+ int otherDoc = iterator.docID();
+ if (otherDoc < doc) {
+ otherDoc = iterator.advance(doc);
+ }
+ if (doc != otherDoc) {
+ doc = lead.advance(otherDoc);
+ continue outer;
+ }
+ }
+
+ double score = 0;
+ for (Scorable scorable : scorables) {
+ score += scorable.score();
+ }
+ scorable.score = (float) score;
+ collector.collect(doc);
+ }
+ doc = lead.nextDoc();
+ }
+ }
+
+ /**
+ * Score a window of doc IDs by computing matches and scores on the lead costly clause, then
+ * iterate other clauses one by one to remove documents that do not match and increase the global
+ * score by the score of the current clause. This is often faster when a minimum competitive score
+ * is set, as score computations can be more efficient (e.g. thanks to vectorization) and because
+ * we can skip advancing other clauses if the global score so far is not high enough for a doc to
+ * have a chance of being competitive.
+ */
+ private void scoreWindowScoreFirst(
LeafCollector collector, Bits acceptDocs, int min, int max, float maxWindowScore)
throws IOException {
if (maxWindowScore < scorable.minCompetitiveScore) {
@@ -120,13 +166,11 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {
docAndScoreAccBuffer.copyFrom(docAndScoreBuffer);
for (int i = 1; i < scorers.length; ++i) {
- if (scorable.minCompetitiveScore > 0) {
- ScorerUtil.filterCompetitiveHits(
- docAndScoreAccBuffer,
- sumOfOtherClauses[i],
- scorable.minCompetitiveScore,
- scorers.length);
- }
+ ScorerUtil.filterCompetitiveHits(
+ docAndScoreAccBuffer,
+ sumOfOtherClauses[i],
+ scorable.minCompetitiveScore,
+ scorers.length);
ScorerUtil.applyRequiredClause(docAndScoreAccBuffer, iterators[i], scorables[i]);
}
(lucene) 05/05: Merge CombinedFieldQuery's WeightedDisiWrapper into DisiWrapper. (#14747)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit f2eb4d9fe2929f7d0439a6c61665f9dd2e2063d8
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Jun 3 07:06:52 2025 +0200
Merge CombinedFieldQuery's WeightedDisiWrapper into DisiWrapper. (#14747)
This aims at simplifying things a bit to make it easier to implement
`CombinedFieldScorer#nextDocsAndScores` later.
---
.../apache/lucene/search/CombinedFieldQuery.java | 25 ++++++----------------
.../java/org/apache/lucene/search/DisiWrapper.java | 13 +++++++++++
2 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
index 42659e3b37e..7cb6cb89f50 100644
--- a/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
@@ -378,8 +378,10 @@ public final class CombinedFieldQuery extends Query implements Accountable {
List<DisiWrapper> wrappers = new ArrayList<>(iterators.size());
for (int i = 0; i < iterators.size(); i++) {
float weight = fields.get(i).weight;
- wrappers.add(
- new WeightedDisiWrapper(new TermScorer(iterators.get(i), simWeight, null), weight));
+ Scorer scorer = new TermScorer(iterators.get(i), simWeight, null);
+ DisiWrapper w = new DisiWrapper(scorer, false, weight);
+ assert w.postingsEnum != null; // needed to access term frequencies
+ wrappers.add(w);
}
// Even though it is called approximation, it is accurate since none of
// the sub iterators are two-phase iterators.
@@ -401,21 +403,6 @@ public final class CombinedFieldQuery extends Query implements Accountable {
}
}
- private static class WeightedDisiWrapper extends DisiWrapper {
- final PostingsEnum postingsEnum;
- final float weight;
-
- WeightedDisiWrapper(Scorer scorer, float weight) {
- super(scorer, false);
- this.weight = weight;
- this.postingsEnum = (PostingsEnum) scorer.iterator();
- }
-
- float freq() throws IOException {
- return weight * postingsEnum.freq();
- }
- }
-
private static class CombinedFieldScorer extends Scorer {
private final DisjunctionDISIApproximation iterator;
private final MultiNormsLeafSimScorer simScorer;
@@ -434,9 +421,9 @@ public final class CombinedFieldQuery extends Query implements Accountable {
float freq() throws IOException {
DisiWrapper w = iterator.topList();
- float freq = ((WeightedDisiWrapper) w).freq();
+ float freq = w.postingsEnum.freq() * w.weight;
for (w = w.next; w != null; w = w.next) {
- freq += ((WeightedDisiWrapper) w).freq();
+ freq += w.postingsEnum.freq() * w.weight;
if (freq < 0) { // overflow
return Integer.MAX_VALUE;
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisiWrapper.java b/lucene/core/src/java/org/apache/lucene/search/DisiWrapper.java
index 5673f16b37b..d3a73556560 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisiWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisiWrapper.java
@@ -17,6 +17,7 @@
package org.apache.lucene.search;
import java.util.Objects;
+import org.apache.lucene.index.PostingsEnum;
/**
* Wrapper used in {@link DisiPriorityQueue}.
@@ -25,6 +26,8 @@ import java.util.Objects;
*/
public class DisiWrapper {
public final DocIdSetIterator iterator;
+ // Same as `iterator` if `iterator` is an instance of PostingsEnum, null otherwise
+ final PostingsEnum postingsEnum;
public final Scorer scorer;
public final Scorable scorable;
public final long cost;
@@ -45,7 +48,14 @@ public class DisiWrapper {
// for MaxScoreBulkScorer
float maxWindowScore;
+ // for CombinedFieldQuery (BM25F)
+ final float weight;
+
public DisiWrapper(Scorer scorer, boolean impacts) {
+ this(scorer, impacts, 1f);
+ }
+
+ DisiWrapper(Scorer scorer, boolean impacts, float weight) {
this.scorer = Objects.requireNonNull(scorer);
this.scorable = ScorerUtil.likelyTermScorer(scorer);
if (impacts) {
@@ -53,6 +63,7 @@ public class DisiWrapper {
} else {
this.iterator = scorer.iterator();
}
+ this.postingsEnum = iterator instanceof PostingsEnum pe ? pe : null;
this.cost = iterator.cost();
this.doc = -1;
this.twoPhaseView = scorer.twoPhaseIterator();
@@ -64,5 +75,7 @@ public class DisiWrapper {
approximation = iterator;
matchCost = 0f;
}
+
+ this.weight = weight;
}
}