You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2026/05/29 19:30:41 UTC
(lucene) branch branch_10x updated: Fix undercounting of RAM used by vectors buffered in in-memory segments (#15982)
This is an automated email from the ASF dual-hosted git repository.
shubhamvishu pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 6d7253df9db Fix undercounting of RAM used by vectors buffered in in-memory segments (#15982)
6d7253df9db is described below
commit 6d7253df9db8e6b93a763eeca1be92bfe2a57bb1
Author: Prithvi S <pr...@gmail.com>
AuthorDate: Sat May 30 00:02:16 2026 +0530
Fix undercounting of RAM used by vectors buffered in in-memory segments (#15982)
---
lucene/CHANGES.txt | 7 +++
.../Lucene102BinaryQuantizedVectorsWriter.java | 25 ++++++--
.../Lucene99ScalarQuantizedVectorsWriter.java | 19 +++++-
.../lucene/codecs/BufferingKnnVectorsWriter.java | 2 +-
.../Lucene104ScalarQuantizedVectorsWriter.java | 27 +++++++--
.../tests/index/BaseKnnVectorsFormatTestCase.java | 69 ++++++++++++++++++++++
6 files changed, 137 insertions(+), 12 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 57e5f107582..87f76d8c5f7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -246,6 +246,13 @@ Bug Fixes
* GITHUB#16046: Fix double-counting of underlying Automaton in CompiledAutomaton#ramBytesUsed. (Eugene Rizhkov)
+* GITHUB#15901: Fix undercounting of RAM used by vectors buffered in in-memory segments.
+ BufferingKnnVectorsWriter hardcoded Float.BYTES for all vector encodings, overcounting byte
+ vectors by 4x. Scalar quantized vector writers (Lucene104, Lucene99, Lucene102) did not
+ account for rawVectorDelegate RAM, making byte vector fields completely invisible in RAM
+ reporting. Also added missing dimensionSums array accounting in quantized writers.
+ (Prithvi S)
+
Other
---------------------
* GITHUB#15586: Document that scoring and ranking may change across major Lucene versions, and that applications requiring stable ranking should explicitly configure Similarity. (Parveen Saini)
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
index 359901fc02f..64afefa2f5c 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
@@ -52,6 +52,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
@@ -459,9 +460,14 @@ class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {
@Override
public long ramBytesUsed() {
long total = SHALLOW_RAM_BYTES_USED;
+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.
+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.
+ total += rawVectorDelegate.ramBytesUsed();
for (FieldWriter field : fields) {
- // the field tracks the delegate field usage
- total += field.ramBytesUsed();
+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter
+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector
+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.
+ total += field.quantizationOverheadBytesUsed();
}
return total;
}
@@ -536,11 +542,22 @@ class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {
throw new UnsupportedOperationException();
}
+ /**
+ * Returns the RAM usage of quantization-specific state only (magnitudes, dimensionSums, shallow
+ * object overhead). The underlying flat vector data is tracked separately by the
+ * rawVectorDelegate at the writer level to avoid double-counting.
+ */
+ long quantizationOverheadBytesUsed() {
+ long size = SHALLOW_SIZE;
+ size += magnitudes.ramBytesUsed();
+ size += RamUsageEstimator.sizeOf(dimensionSums);
+ return size;
+ }
+
@Override
public long ramBytesUsed() {
- long size = SHALLOW_SIZE;
+ long size = quantizationOverheadBytesUsed();
size += flatFieldVectorsWriter.ramBytesUsed();
- size += magnitudes.ramBytesUsed();
return size;
}
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
index f366e2b851b..a726ce891fd 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
@@ -286,9 +286,14 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
@Override
public long ramBytesUsed() {
long total = SHALLOW_RAM_BYTES_USED;
+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.
+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.
+ total += rawVectorDelegate.ramBytesUsed();
for (FieldWriter field : fields) {
- // the field tracks the delegate field usage
- total += field.ramBytesUsed();
+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter
+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector
+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.
+ total += field.quantizationOverheadBytesUsed();
}
return total;
}
@@ -733,9 +738,17 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
return quantizer;
}
+ /**
+ * Returns the RAM usage of quantization-specific state only. The underlying flat vector data is
+ * tracked separately by the rawVectorDelegate at the writer level.
+ */
+ long quantizationOverheadBytesUsed() {
+ return SHALLOW_SIZE;
+ }
+
@Override
public long ramBytesUsed() {
- long size = SHALLOW_SIZE;
+ long size = quantizationOverheadBytesUsed();
size += flatFieldVectorsWriter.ramBytesUsed();
return size;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java
index 32df27bad71..629afec1ec1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BufferingKnnVectorsWriter.java
@@ -262,7 +262,7 @@ public abstract class BufferingKnnVectorsWriter extends KnnVectorsWriter {
* (long)
(RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER)
- + vectors.size() * (long) dim * Float.BYTES;
+ + vectors.size() * (long) dim * fieldInfo.getVectorEncoding().byteSize;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
index ddb13e36680..fd60dc5a4a1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
@@ -49,6 +49,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
@@ -475,9 +476,16 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
@Override
public long ramBytesUsed() {
long total = SHALLOW_RAM_BYTES_USED;
+ // The rawVectorDelegate tracks all vector data for both byte and float32 fields.
+ // For byte vector fields (which bypass our FieldWriter), this is the only accounting.
+ // For float32 fields, this covers the flat vector data; our FieldWriter adds the
+ // quantization-specific overhead (magnitudes, dimensionSums) on top.
+ total += rawVectorDelegate.ramBytesUsed();
for (FieldWriter field : fields) {
- // the field tracks the delegate field usage
- total += field.ramBytesUsed();
+ // quantizationOverheadBytesUsed() intentionally excludes flatFieldVectorsWriter
+ // because rawVectorDelegate.ramBytesUsed() already accounts for all flat vector
+ // data at the writer level. Calling field.ramBytesUsed() here would double-count.
+ total += field.quantizationOverheadBytesUsed();
}
return total;
}
@@ -552,11 +560,22 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
throw new UnsupportedOperationException();
}
+ /**
+ * Returns the RAM usage of quantization-specific state only (magnitudes, dimensionSums, shallow
+ * object overhead). The underlying flat vector data is tracked separately by the
+ * rawVectorDelegate at the writer level to avoid double-counting.
+ */
+ long quantizationOverheadBytesUsed() {
+ long size = SHALLOW_SIZE;
+ size += magnitudes.ramBytesUsed();
+ size += RamUsageEstimator.sizeOf(dimensionSums);
+ return size;
+ }
+
@Override
public long ramBytesUsed() {
- long size = SHALLOW_SIZE;
+ long size = quantizationOverheadBytesUsed();
size += flatFieldVectorsWriter.ramBytesUsed();
- size += magnitudes.ramBytesUsed();
return size;
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index 9e025ba86de..b207406f3ad 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -434,6 +434,75 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
dir.close();
}
+ @SuppressWarnings("unchecked")
+ public void testWriterByteVectorRamEstimate() throws Exception {
+ final FieldInfos fieldInfos = new FieldInfos(new FieldInfo[0]);
+ final Directory dir = newDirectory();
+ Codec codec = Codec.getDefault();
+ final SegmentInfo si =
+ new SegmentInfo(
+ dir,
+ Version.LATEST,
+ Version.LATEST,
+ "0",
+ 10000,
+ false,
+ false,
+ codec,
+ Collections.emptyMap(),
+ StringHelper.randomId(),
+ new HashMap<>(),
+ null);
+ final SegmentWriteState state =
+ new SegmentWriteState(
+ InfoStream.getDefault(), dir, si, fieldInfos, null, newIOContext(random()));
+ final KnnVectorsFormat format = codec.knnVectorsFormat();
+ try (KnnVectorsWriter writer = format.fieldsWriter(state)) {
+ int dim = random().nextInt(64) + 1;
+ if (dim % 2 == 1) {
+ ++dim;
+ }
+ int numDocs = atLeast(100);
+ KnnFieldVectorsWriter<byte[]> fieldWriter =
+ (KnnFieldVectorsWriter<byte[]>)
+ writer.addField(
+ new FieldInfo(
+ "fieldA",
+ 0,
+ false,
+ false,
+ false,
+ IndexOptions.NONE,
+ DocValuesType.NONE,
+ DocValuesSkipIndexType.NONE,
+ -1,
+ Map.of(),
+ 0,
+ 0,
+ 0,
+ dim,
+ VectorEncoding.BYTE,
+ VectorSimilarityFunction.DOT_PRODUCT,
+ false,
+ false));
+ for (int i = 0; i < numDocs; i++) {
+ fieldWriter.addValue(i, randomVector8(dim));
+ }
+ // Validate the field-level RAM accounting uses correct byte sizes.
+ // The reported RAM must be at least the raw byte vector data.
+ final long fieldRamBytesUsed = fieldWriter.ramBytesUsed();
+ final long rawByteVectorData = (long) dim * numDocs * Byte.BYTES;
+ assertTrue(
+ "Expected field ramBytesUsed ("
+ + fieldRamBytesUsed
+ + ") >= raw byte vector data size ("
+ + rawByteVectorData
+ + ")",
+ fieldRamBytesUsed >= rawByteVectorData);
+ }
+ dir.close();
+ }
+
public void testIllegalSimilarityFunctionChangeTwoWriters() throws Exception {
try (Directory dir = newDirectory()) {
try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {