You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2025/07/16 07:43:47 UTC
(lucene) branch branch_10x updated: Updating Dense#intoBitSet to properly set upTo if it exceeds bitset size (#14922)
This is an automated email from the ASF dual-hosted git repository.
guofeng pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 529dd0acabd Updating Dense#intoBitSet to properly set upTo if it exceeds bitset size (#14922)
529dd0acabd is described below
commit 529dd0acabde5c6107b316359cdc930c7a4fe6e3
Author: Panagiotis Bailis <pm...@gmail.com>
AuthorDate: Wed Jul 16 10:37:53 2025 +0300
Updating Dense#intoBitSet to properly set upTo if it exceeds bitset size (#14922)
---
lucene/CHANGES.txt | 3 +++
.../apache/lucene/codecs/lucene90/IndexedDISI.java | 24 ++++++++++++-----
.../lucene/codecs/lucene90/TestIndexedDISI.java | 30 ++++++++++++++++++++++
3 files changed, 51 insertions(+), 6 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index fe009a92f11..d60be30f5bb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -137,6 +137,9 @@ Bug Fixes
* GITHUB#14838: Make it possible to extend Patience/Seeded knn queries (Tommaso Teofili)
+* GITHUB#14922: Fix for IndexedDISI.Dense#intoBitSetWithinBlock to take into account the provided bitset size
+ and avoid throwing IndexOutOfBoundsException (Panagiotis Bailis)
+
Build
---------------------
* Upgrade forbiddenapis to version 3.9. (Uwe Schindler)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java
index 1d64be1a928..83350e5ce95 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java
@@ -27,6 +27,7 @@ import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.MathUtil;
import org.apache.lucene.util.RoaringDocIdSet;
/**
@@ -473,7 +474,7 @@ public final class IndexedDISI extends AbstractDocIdSetIterator {
@Override
public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
- assert doc >= offset;
+ assert doc >= offset : "offset=" + offset + " doc=" + doc;
while (doc < upTo && method.intoBitSetWithinBlock(this, upTo, bitSet, offset) == false) {
readBlockHeader();
boolean found = method.advanceWithinBlock(this, block);
@@ -719,10 +720,10 @@ public final class IndexedDISI extends AbstractDocIdSetIterator {
if (disi.bitSet == null) {
disi.bitSet = new FixedBitSet(BLOCK_SIZE);
}
-
- int sourceFrom = disi.doc & 0xFFFF;
- int sourceTo = Math.min(upTo - disi.block, BLOCK_SIZE);
int destFrom = disi.doc - offset;
+ int destTo = MathUtil.unsignedMin(upTo, offset + bitSet.length());
+ int sourceFrom = disi.doc & 0xFFFF;
+ int sourceTo = Math.min(destTo - disi.block, BLOCK_SIZE);
long fp = disi.slice.getFilePointer();
disi.slice.seek(fp - Long.BYTES); // seek back a long to include current word (disi.word).
@@ -731,13 +732,24 @@ public final class IndexedDISI extends AbstractDocIdSetIterator {
FixedBitSet.orRange(disi.bitSet, sourceFrom, bitSet, destFrom, sourceTo - sourceFrom);
int blockEnd = disi.block | 0xFFFF;
- if (upTo > blockEnd) {
+ if (destTo > blockEnd) {
disi.slice.seek(disi.blockEnd);
disi.index += disi.bitSet.cardinality(sourceFrom, sourceTo);
return false;
} else {
disi.slice.seek(fp);
- return advanceWithinBlock(disi, upTo);
+ boolean found = advanceWithinBlock(disi, destTo);
+ if (found && disi.doc < upTo) {
+ throw new IllegalStateException(
+ "There are bits set in the source bitset that are not accounted for."
+ + " doc="
+ + disi.doc
+ + " upTo="
+ + upTo
+ + " disi.block="
+ + disi.block);
+ }
+ return found;
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestIndexedDISI.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestIndexedDISI.java
index 135019fe387..974abff9f24 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestIndexedDISI.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestIndexedDISI.java
@@ -388,6 +388,36 @@ public class TestIndexedDISI extends LuceneTestCase {
}
}
+ public void testDenseBitSizeLessThanBlockSize() throws IOException {
+ final byte denseRankPower = (byte) (random().nextInt(7) + 7);
+ try (Directory dir = newDirectory()) {
+ // initialize a maxDoc that is less than IndexedDISI.BLOCK_SIZE
+ int maxDoc = random().nextInt(4096 * 2, 65536);
+ FixedBitSet set = new FixedBitSet(maxDoc);
+ for (int i = 0; i < maxDoc; i += 2) { // Set every other to ensure dense
+ set.set(i);
+ }
+ int jumpTableEntryCount; // this should always be 0 given that maxDoc < BLOCK_SIZE
+ long length;
+ try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
+ jumpTableEntryCount =
+ IndexedDISI.writeBitSet(
+ new BitSetIterator(set, set.cardinality()), out, denseRankPower);
+ length = out.getFilePointer();
+ assertTrue(
+ "jumpTableEntryCount should be 0 for dense bitsets with size < BLOCK_SIZE",
+ 0 == jumpTableEntryCount);
+ }
+ try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
+ IndexedDISI disi =
+ new IndexedDISI(in, 0L, length, jumpTableEntryCount, denseRankPower, set.cardinality());
+ FixedBitSet disiSet = new FixedBitSet(maxDoc);
+ // This would throw IOOB if bitset size is not handled correctly as per #14882
+ disiSet.or(disi);
+ }
+ }
+ }
+
public void testIllegalDenseRankPower() throws IOException {
// Legal values