You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ka...@apache.org on 2026/02/04 14:43:42 UTC
(lucene) branch branch_10_4 updated: Fix MergedByteVectorValues internal ordinal tracking (#15553)
This is an automated email from the ASF dual-hosted git repository.
kaivalnp pushed a commit to branch branch_10_4
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10_4 by this push:
new 62c97771f2e Fix MergedByteVectorValues internal ordinal tracking (#15553)
62c97771f2e is described below
commit 62c97771f2e3c4ef19181d4aab2fafb2a50a54f9
Author: Finn <67...@users.noreply.github.com>
AuthorDate: Tue Feb 3 21:55:29 2026 +0000
Fix MergedByteVectorValues internal ordinal tracking (#15553)
Move internal ordinal tracking in `MergedByteVectorValues` from `vectorValue` -> `nextDoc` to allow loading only a subset of vectors during iteration.
---
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/codecs/KnnVectorsWriter.java | 36 +++++++---
.../lucene/codecs/TestMergedVectorValues.java | 84 ++++++++++++++++++++++
3 files changed, 115 insertions(+), 8 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9e280525025..a1170cf66f7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -210,6 +210,9 @@ Bug Fixes
* GITHUB#15554: Fix tessellator failure by preferring the shared vertex that is the leftmost vertex of the hole
(Ignacio Vera)
+* GITHUB#15553: Fix MergedByteVectorValues lastOrd behavior to enable partitioning of vector values
+(Finn Roblin, Dooyong Kim)
+
Other
---------------------
* GITHUB#15237: Fix SmartChinese to only deserialize dictionary data from classpath with a native array
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java
index 4bd10215c25..5eaa6da4a8a 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java
@@ -121,8 +121,12 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
}
}
- /** Tracks state of one sub-reader that we are merging */
- private static class FloatVectorValuesSub extends DocIDMerger.Sub {
+ /**
+ * Tracks state of one sub-reader of float vectors that we are merging.
+ *
+ * @lucene.internal
+ */
+ static class FloatVectorValuesSub extends DocIDMerger.Sub {
final FloatVectorValues values;
final KnnVectorValues.DocIndexIterator iterator;
@@ -144,7 +148,12 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
}
}
- private static class ByteVectorValuesSub extends DocIDMerger.Sub {
+ /**
+ * Tracks state of one sub-reader of byte vectors that we are merging.
+ *
+ * @lucene.internal
+ */
+ static class ByteVectorValuesSub extends DocIDMerger.Sub {
final ByteVectorValues values;
final KnnVectorValues.DocIndexIterator iterator;
@@ -303,6 +312,11 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
mergeState);
}
+ /**
+ * Unified view over several segments containing float vector values.
+ *
+ * @lucene.internal
+ */
static class MergedFloat32VectorValues extends FloatVectorValues {
private final List<FloatVectorValuesSub> subs;
private final DocIDMerger<FloatVectorValuesSub> docIdMerger;
@@ -311,7 +325,8 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
private int lastOrd = -1;
FloatVectorValuesSub current;
- private MergedFloat32VectorValues(List<FloatVectorValuesSub> subs, MergeState mergeState)
+ // package-private for testing
+ MergedFloat32VectorValues(List<FloatVectorValuesSub> subs, MergeState mergeState)
throws IOException {
this.subs = subs;
docIdMerger = DocIDMerger.of(subs, mergeState.needsIndexSort);
@@ -401,6 +416,11 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
}
}
+ /**
+ * Unified view over several segments containing byte vector values.
+ *
+ * @lucene.internal
+ */
static class MergedByteVectorValues extends ByteVectorValues {
private final List<ByteVectorValuesSub> subs;
private final DocIDMerger<ByteVectorValuesSub> docIdMerger;
@@ -410,7 +430,8 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
private int docId = -1;
ByteVectorValuesSub current;
- private MergedByteVectorValues(List<ByteVectorValuesSub> subs, MergeState mergeState)
+ // package-private for testing
+ MergedByteVectorValues(List<ByteVectorValuesSub> subs, MergeState mergeState)
throws IOException {
this.subs = subs;
docIdMerger = DocIDMerger.of(subs, mergeState.needsIndexSort);
@@ -423,11 +444,9 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
@Override
public byte[] vectorValue(int ord) throws IOException {
- if (ord != lastOrd + 1) {
+ if (ord != lastOrd) {
throw new IllegalStateException(
"only supports forward iteration: ord=" + ord + ", lastOrd=" + lastOrd);
- } else {
- lastOrd = ord;
}
return current.values.vectorValue(current.index());
}
@@ -455,6 +474,7 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
index = NO_MORE_DOCS;
} else {
docId = current.mappedDocID;
+ ++lastOrd;
++index;
}
return docId;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java b/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java
new file mode 100644
index 00000000000..5f1fc6a2267
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.lucene.index.ByteVectorValues;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.MergeState;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+/** Tests for merged vector values to ensure lastOrd is properly incremented during iteration. */
+public class TestMergedVectorValues extends LuceneTestCase {
+
+ /**
+ * Test that skipping vectors in MergedByteVectorValues via nextDoc() and then loading a
+ * subsequent vector via vectorValue() works correctly.
+ */
+ public void testSkipsInMergedByteVectorValues() throws IOException {
+ // Data
+ List<byte[]> vectors = List.of(new byte[] {0}, new byte[] {1});
+
+ // Setup
+ KnnVectorsWriter.ByteVectorValuesSub sub =
+ new KnnVectorsWriter.ByteVectorValuesSub(x -> x, ByteVectorValues.fromBytes(vectors, 1));
+ MergeState state =
+ new MergeState(
+ null, null, null, null, null, null, null, null, null, null, null, null, null, null,
+ null, false);
+
+ // Run the test
+ ByteVectorValues values =
+ new KnnVectorsWriter.MergedVectorValues.MergedByteVectorValues(List.of(sub), state);
+
+ // Skip doc 0 and load doc 1
+ values.iterator().nextDoc(); // doc 0
+ values.iterator().nextDoc(); // doc 1
+
+ // Read vector for doc 1
+ assertArrayEquals(vectors.get(1), values.vectorValue(1));
+ }
+
+ /**
+ * Test that skipping vectors in MergedFloat32VectorValues via nextDoc() and then loading a
+ * subsequent vector via vectorValue() works correctly.
+ */
+ public void testSkipsInMergedFloat32VectorValues() throws IOException {
+ // Data
+ List<float[]> vectors = List.of(new float[] {0.0f}, new float[] {1.0f});
+
+ // Setup
+ KnnVectorsWriter.FloatVectorValuesSub sub =
+ new KnnVectorsWriter.FloatVectorValuesSub(x -> x, FloatVectorValues.fromFloats(vectors, 1));
+ MergeState state =
+ new MergeState(
+ null, null, null, null, null, null, null, null, null, null, null, null, null, null,
+ null, false);
+
+ // Run the test
+ FloatVectorValues values =
+ new KnnVectorsWriter.MergedVectorValues.MergedFloat32VectorValues(List.of(sub), state);
+
+ // Skip doc 0 and load doc 1
+ values.iterator().nextDoc(); // doc 0
+ values.iterator().nextDoc(); // doc 1
+
+ // Read vector for doc 1
+ assertArrayEquals(vectors.get(1), values.vectorValue(1), 0.0f);
+ }
+}