You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2026/08/03 16:03:52 UTC
(lucene) branch main updated: Wrap dropped-FP scalar-quantized float values so scoring stays quantized (#16415)
This is an automated email from the ASF dual-hosted git repository.
msokolov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 7621eae9a22 Wrap dropped-FP scalar-quantized float values so scoring stays quantized (#16415)
7621eae9a22 is described below
commit 7621eae9a22669ff1082eec4c14a6d40079f9256
Author: Pulkit Gupta <36...@users.noreply.github.com>
AuthorDate: Mon Aug 3 12:03:45 2026 -0400
Wrap dropped-FP scalar-quantized float values so scoring stays quantized (#16415)
---
.../Lucene99ScalarQuantizedVectorsReader.java | 34 +++++----
.../Lucene104ScalarQuantizedVectorsReader.java | 36 +++++----
.../tests/index/BaseKnnVectorsFormatTestCase.java | 87 ++++++++++++++++++++++
3 files changed, 130 insertions(+), 27 deletions(-)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index 6c4b7e402e9..d2cb4e2b38a 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -195,19 +195,6 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
public FloatVectorValues getFloatVectorValues(String field) throws IOException {
final FieldEntry fieldEntry = getFieldEntry(field);
final FloatVectorValues rawVectorValues = rawVectorsReader.getFloatVectorValues(field);
- if (rawVectorValues.size() == 0) {
- return OffHeapQuantizedFloatVectorValues.load(
- fieldEntry.ordToDoc,
- fieldEntry.dimension,
- fieldEntry.size,
- fieldEntry.scalarQuantizer,
- fieldEntry.similarityFunction,
- vectorScorer,
- fieldEntry.compress,
- fieldEntry.vectorDataOffset,
- fieldEntry.vectorDataLength,
- quantizedVectorData);
- }
OffHeapQuantizedByteVectorValues quantizedByteVectorValues =
OffHeapQuantizedByteVectorValues.load(
@@ -221,6 +208,27 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
fieldEntry.vectorDataOffset,
fieldEntry.vectorDataLength,
quantizedVectorData);
+
+ if (rawVectorValues.size() == 0) {
+ // Full-precision vectors were dropped. Wrap the dequantizing read view with the quantized
+ // values so scorer() stays quantized (as in the full-precision branch) while
+ // vectorValue()/rescorer() dequantize. Passing the dequantized view straight to a scorer
+ // would misroute to the non-quantized flat scorer over the quantized slice.
+ FloatVectorValues dequantizedRawVectorValues =
+ OffHeapQuantizedFloatVectorValues.load(
+ fieldEntry.ordToDoc,
+ fieldEntry.dimension,
+ fieldEntry.size,
+ fieldEntry.scalarQuantizer,
+ fieldEntry.similarityFunction,
+ vectorScorer,
+ fieldEntry.compress,
+ fieldEntry.vectorDataOffset,
+ fieldEntry.vectorDataLength,
+ quantizedVectorData);
+ return new QuantizedVectorValues(dequantizedRawVectorValues, quantizedByteVectorValues);
+ }
+
return new QuantizedVectorValues(rawVectorValues, quantizedByteVectorValues);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index ef458b05162..79ff5235edc 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -251,20 +251,6 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
FloatVectorValues rawFloatVectorValues = rawVectorsReader.getFloatVectorValues(field);
- if (rawFloatVectorValues.size() == 0) {
- return OffHeapScalarQuantizedFloatVectorValues.load(
- fi.ordToDocDISIReaderConfiguration,
- fi.dimension,
- fi.size,
- fi.scalarEncoding,
- fi.similarityFunction,
- vectorScorer,
- fi.centroid,
- fi.vectorDataOffset,
- fi.vectorDataLength,
- quantizedVectorData);
- }
-
OffHeapScalarQuantizedVectorValues sqvv =
OffHeapScalarQuantizedVectorValues.load(
fi.ordToDocDISIReaderConfiguration,
@@ -279,6 +265,28 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
fi.vectorDataOffset,
fi.vectorDataLength,
quantizedVectorData);
+
+ if (rawFloatVectorValues.size() == 0) {
+ // Full-precision vectors were dropped. Wrap the dequantizing read view with sqvv so scorer()
+ // stays quantized (as in the full-precision branch) while vectorValue()/rescorer()
+ // dequantize.
+ // Passing the dequantized view straight to a scorer would misroute to the non-quantized flat
+ // scorer over the quantized slice.
+ FloatVectorValues dequantizedRawVectorValues =
+ OffHeapScalarQuantizedFloatVectorValues.load(
+ fi.ordToDocDISIReaderConfiguration,
+ fi.dimension,
+ fi.size,
+ fi.scalarEncoding,
+ fi.similarityFunction,
+ vectorScorer,
+ fi.centroid,
+ fi.vectorDataOffset,
+ fi.vectorDataLength,
+ quantizedVectorData);
+ return new ScalarQuantizedVectorValues(dequantizedRawVectorValues, sqvv);
+ }
+
return new ScalarQuantizedVectorValues(rawFloatVectorValues, sqvv);
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index bc3b93a77db..d5097cf411f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -2135,6 +2135,93 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
}
}
+ /**
+ * Tests that dropping the raw full-precision vectors does not change scoring. {@link
+ * FloatVectorValues#scorer(float[])} is documented to score against the quantized vectors when
+ * the underlying format quantizes, so a quantized index must produce identical scores before and
+ * after its raw vector file is emptied.
+ */
+ public void testScoresUnchangedWithEmptyRawVectors() throws Exception {
+ assumeTrue("Test only applies to scalar quantized formats", supportsFloatVectorFallback());
+
+ String vectorFieldName = "vec1";
+ int numVectors = 1 + random().nextInt(50);
+ int dim = random().nextInt(64) + 1;
+ if (dim % 2 == 1) {
+ dim++;
+ }
+ VectorSimilarityFunction similarityFunction = randomSimilarity();
+ List<float[]> vectors = getRandomFloatVector(numVectors, dim, false);
+ float[] query = getRandomFloatVector(1, dim, false).get(0);
+
+ try (BaseDirectoryWrapper dir = newDirectory()) {
+ // The raw vector file is emptied below, which CheckIndex rightly rejects.
+ dir.setCheckIndexOnClose(false);
+
+ try (IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig()
+ .setMaxBufferedDocs(numVectors + 1)
+ .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
+ .setMergePolicy(NoMergePolicy.INSTANCE)
+ .setUseCompoundFile(false)
+ .setCodec(getCodecForFloatVectorFallbackTest()))) {
+ for (int i = 0; i < numVectors; i++) {
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField(vectorFieldName, vectors.get(i), similarityFunction));
+ w.addDocument(doc);
+ }
+ }
+
+ // Scores while the raw full-precision vectors are still present.
+ Map<Integer, Float> expectedScores = scoreAllDocs(dir, vectorFieldName, query);
+ assertEquals("expected every document to be scored", numVectors, expectedScores.size());
+
+ simulateEmptyRawVectors(dir);
+
+ // Both reads are expected to score against the same quantized vectors, so the scores must
+ // match exactly rather than merely within a quantization-error tolerance.
+ Map<Integer, Float> actualScores = scoreAllDocs(dir, vectorFieldName, query);
+ assertEquals(expectedScores.keySet(), actualScores.keySet());
+ for (Map.Entry<Integer, Float> entry : expectedScores.entrySet()) {
+ assertEquals(
+ "score changed for doc " + entry.getKey() + " after dropping raw vectors",
+ entry.getValue(),
+ actualScores.get(entry.getKey()),
+ 0f);
+ }
+ }
+ }
+
+ /**
+ * Scores every document holding a vector for {@code field} against {@code query}, keyed by doc
+ * id. Keying by doc id rather than ordinal keeps the comparison meaningful even when a different
+ * {@link FloatVectorValues} implementation backs the iterator.
+ */
+ private Map<Integer, Float> scoreAllDocs(Directory dir, String field, float[] query)
+ throws IOException {
+ Map<Integer, Float> scores = new HashMap<>();
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ if (leafReader instanceof CodecReader codecReader) {
+ KnnVectorsReader knnVectorsReader =
+ codecReader.getVectorReader().unwrapReaderForField(field);
+ FloatVectorValues floatVectorValues = knnVectorsReader.getFloatVectorValues(field);
+ assertNotNull(floatVectorValues);
+ VectorScorer scorer = floatVectorValues.scorer(query);
+ assertNotNull(scorer);
+ DocIdSetIterator iterator = scorer.iterator();
+ for (int doc = iterator.nextDoc(); doc != NO_MORE_DOCS; doc = iterator.nextDoc()) {
+ scores.put(doc, scorer.score());
+ }
+ } else {
+ fail("reader is not CodecReader");
+ }
+ }
+ return scores;
+ }
+
/**
* Simulates empty raw vectors by modifying index files. Override in codecs that support
* FloatVector fallback.