You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by go...@apache.org on 2026/10/22 07:40:32 UTC
(pinot) branch master updated: Make FixedBitMVForwardIndexReader.getNumValuesMV advance the reader context (#19992)
This is an automated email from the ASF dual-hosted git repository.
gortiz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 24c520be6e0 Make FixedBitMVForwardIndexReader.getNumValuesMV advance the reader context (#19992)
24c520be6e0 is described below
commit 24c520be6e063f0253a97902051b6853b42bf1bb
Author: Gonzalo Ortiz Jaureguizar <go...@users.noreply.github.com>
AuthorDate: Thu Oct 22 09:40:24 2026 +0200
Make FixedBitMVForwardIndexReader.getNumValuesMV advance the reader context (#19992)
---
.../forward/FixedBitMVForwardIndexReader.java | 5 ++
.../index/forward/FixedBitMVForwardIndexTest.java | 69 ++++++++++++++++++++++
2 files changed, 74 insertions(+)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
index b690fdf0d37..f927e191512 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
@@ -199,6 +199,11 @@ public final class FixedBitMVForwardIndexReader implements ForwardIndexReader<Fi
} else {
endIndex = _bitmapReader.getNextSetBitOffset(startIndex + 1);
}
+ // Record the position like getDictIdMV() does. Without it, every call for a sequential scan takes the bitmap
+ // scan from the context position (or the chunk start) instead of the O(1) next-doc path, which makes counting
+ // the values of a whole segment quadratic in the number of docs per chunk.
+ context._docId = docId;
+ context._endOffset = endIndex;
return endIndex - startIndex;
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java
index 2b35db720de..f39d73df000 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/FixedBitMVForwardIndexTest.java
@@ -113,6 +113,75 @@ public class FixedBitMVForwardIndexTest implements PinotBuffersAfterMethodCheckR
}
}
+ /// Mixes getNumValuesMV() and getDictIdMV() calls on one context, in sequential, skipping and random doc order.
+ /// Few values per doc give about a thousand docs per chunk, so the calls cross chunks and also move inside one.
+ @Test
+ public void testNumValuesMVSharesContextWithDictIdMV()
+ throws Exception {
+ int numDocs = 5000;
+ int numBitsPerValue = 8;
+ int[][] valuesArray = new int[numDocs][];
+ int totalNumValues = 0;
+ for (int i = 0; i < numDocs; i++) {
+ int[] values = new int[RANDOM.nextInt(3) + 1];
+ for (int j = 0; j < values.length; j++) {
+ values[j] = RANDOM.nextInt(1 << numBitsPerValue);
+ }
+ valuesArray[i] = values;
+ totalNumValues += values.length;
+ }
+ try (FixedBitMVForwardIndexWriter writer = new FixedBitMVForwardIndexWriter(INDEX_FILE, numDocs, totalNumValues,
+ numBitsPerValue)) {
+ for (int[] values : valuesArray) {
+ writer.putDictIds(values);
+ }
+ }
+
+ int[] randomOrder = new int[numDocs];
+ for (int i = 0; i < numDocs; i++) {
+ randomOrder[i] = RANDOM.nextInt(numDocs);
+ }
+ try (PinotDataBuffer dataBuffer = PinotDataBuffer.mapReadOnlyBigEndianFile(INDEX_FILE);
+ FixedBitMVForwardIndexReader reader = new FixedBitMVForwardIndexReader(dataBuffer, numDocs, totalNumValues,
+ numBitsPerValue)) {
+ int[] valueBuffer = new int[3];
+ // Sequential, counting only
+ try (FixedBitMVForwardIndexReader.Context context = reader.createContext()) {
+ for (int i = 0; i < numDocs; i++) {
+ assertEquals(reader.getNumValuesMV(i, context), valuesArray[i].length, "docId: " + i);
+ }
+ }
+ // Sequential, counting and then reading the same doc
+ try (FixedBitMVForwardIndexReader.Context context = reader.createContext()) {
+ for (int i = 0; i < numDocs; i++) {
+ assertEquals(reader.getNumValuesMV(i, context), valuesArray[i].length, "docId: " + i);
+ assertEquals(reader.getDictIdMV(i, context), valuesArray[i], "docId: " + i);
+ }
+ }
+ // Skipping docs, alternating the two calls
+ try (FixedBitMVForwardIndexReader.Context context = reader.createContext()) {
+ for (int i = 0; i < numDocs; i += 3) {
+ if (i % 2 == 0) {
+ assertEquals(reader.getNumValuesMV(i, context), valuesArray[i].length, "docId: " + i);
+ } else {
+ int numValues = reader.getDictIdMV(i, valueBuffer, context);
+ assertEquals(numValues, valuesArray[i].length, "docId: " + i);
+ }
+ }
+ }
+ // Random order, including going backwards
+ try (FixedBitMVForwardIndexReader.Context context = reader.createContext()) {
+ for (int docId : randomOrder) {
+ assertEquals(reader.getNumValuesMV(docId, context), valuesArray[docId].length, "docId: " + docId);
+ if (docId % 2 == 0) {
+ assertEquals(reader.getDictIdMV(docId, context), valuesArray[docId], "docId: " + docId);
+ }
+ }
+ }
+ }
+ FileUtils.forceDelete(INDEX_FILE);
+ }
+
@AfterClass
public void tearDown()
throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org