You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2026/08/03 16:03:52 UTC

(lucene) branch main updated: Wrap dropped-FP scalar-quantized float values so scoring stays quantized (#16415)

This is an automated email from the ASF dual-hosted git repository.

msokolov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new 7621eae9a22 Wrap dropped-FP scalar-quantized float values so scoring stays quantized (#16415)
7621eae9a22 is described below

commit 7621eae9a22669ff1082eec4c14a6d40079f9256
Author: Pulkit Gupta <36...@users.noreply.github.com>
AuthorDate: Mon Aug 3 12:03:45 2026 -0400

    Wrap dropped-FP scalar-quantized float values so scoring stays quantized (#16415)
---
 .../Lucene99ScalarQuantizedVectorsReader.java      | 34 +++++----
 .../Lucene104ScalarQuantizedVectorsReader.java     | 36 +++++----
 .../tests/index/BaseKnnVectorsFormatTestCase.java  | 87 ++++++++++++++++++++++
 3 files changed, 130 insertions(+), 27 deletions(-)

diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index 6c4b7e402e9..d2cb4e2b38a 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -195,19 +195,6 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
   public FloatVectorValues getFloatVectorValues(String field) throws IOException {
     final FieldEntry fieldEntry = getFieldEntry(field);
     final FloatVectorValues rawVectorValues = rawVectorsReader.getFloatVectorValues(field);
-    if (rawVectorValues.size() == 0) {
-      return OffHeapQuantizedFloatVectorValues.load(
-          fieldEntry.ordToDoc,
-          fieldEntry.dimension,
-          fieldEntry.size,
-          fieldEntry.scalarQuantizer,
-          fieldEntry.similarityFunction,
-          vectorScorer,
-          fieldEntry.compress,
-          fieldEntry.vectorDataOffset,
-          fieldEntry.vectorDataLength,
-          quantizedVectorData);
-    }
 
     OffHeapQuantizedByteVectorValues quantizedByteVectorValues =
         OffHeapQuantizedByteVectorValues.load(
@@ -221,6 +208,27 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
             fieldEntry.vectorDataOffset,
             fieldEntry.vectorDataLength,
             quantizedVectorData);
+
+    if (rawVectorValues.size() == 0) {
+      // Full-precision vectors were dropped. Wrap the dequantizing read view with the quantized
+      // values so scorer() stays quantized (as in the full-precision branch) while
+      // vectorValue()/rescorer() dequantize. Passing the dequantized view straight to a scorer
+      // would misroute to the non-quantized flat scorer over the quantized slice.
+      FloatVectorValues dequantizedRawVectorValues =
+          OffHeapQuantizedFloatVectorValues.load(
+              fieldEntry.ordToDoc,
+              fieldEntry.dimension,
+              fieldEntry.size,
+              fieldEntry.scalarQuantizer,
+              fieldEntry.similarityFunction,
+              vectorScorer,
+              fieldEntry.compress,
+              fieldEntry.vectorDataOffset,
+              fieldEntry.vectorDataLength,
+              quantizedVectorData);
+      return new QuantizedVectorValues(dequantizedRawVectorValues, quantizedByteVectorValues);
+    }
+
     return new QuantizedVectorValues(rawVectorValues, quantizedByteVectorValues);
   }
 
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index ef458b05162..79ff5235edc 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -251,20 +251,6 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
 
     FloatVectorValues rawFloatVectorValues = rawVectorsReader.getFloatVectorValues(field);
 
-    if (rawFloatVectorValues.size() == 0) {
-      return OffHeapScalarQuantizedFloatVectorValues.load(
-          fi.ordToDocDISIReaderConfiguration,
-          fi.dimension,
-          fi.size,
-          fi.scalarEncoding,
-          fi.similarityFunction,
-          vectorScorer,
-          fi.centroid,
-          fi.vectorDataOffset,
-          fi.vectorDataLength,
-          quantizedVectorData);
-    }
-
     OffHeapScalarQuantizedVectorValues sqvv =
         OffHeapScalarQuantizedVectorValues.load(
             fi.ordToDocDISIReaderConfiguration,
@@ -279,6 +265,28 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
             fi.vectorDataOffset,
             fi.vectorDataLength,
             quantizedVectorData);
+
+    if (rawFloatVectorValues.size() == 0) {
+      // Full-precision vectors were dropped. Wrap the dequantizing read view with sqvv so scorer()
+      // stays quantized (as in the full-precision branch) while vectorValue()/rescorer()
+      // dequantize.
+      // Passing the dequantized view straight to a scorer would misroute to the non-quantized flat
+      // scorer over the quantized slice.
+      FloatVectorValues dequantizedRawVectorValues =
+          OffHeapScalarQuantizedFloatVectorValues.load(
+              fi.ordToDocDISIReaderConfiguration,
+              fi.dimension,
+              fi.size,
+              fi.scalarEncoding,
+              fi.similarityFunction,
+              vectorScorer,
+              fi.centroid,
+              fi.vectorDataOffset,
+              fi.vectorDataLength,
+              quantizedVectorData);
+      return new ScalarQuantizedVectorValues(dequantizedRawVectorValues, sqvv);
+    }
+
     return new ScalarQuantizedVectorValues(rawFloatVectorValues, sqvv);
   }
 
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index bc3b93a77db..d5097cf411f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -2135,6 +2135,93 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
     }
   }
 
+  /**
+   * Tests that dropping the raw full-precision vectors does not change scoring. {@link
+   * FloatVectorValues#scorer(float[])} is documented to score against the quantized vectors when
+   * the underlying format quantizes, so a quantized index must produce identical scores before and
+   * after its raw vector file is emptied.
+   */
+  public void testScoresUnchangedWithEmptyRawVectors() throws Exception {
+    assumeTrue("Test only applies to scalar quantized formats", supportsFloatVectorFallback());
+
+    String vectorFieldName = "vec1";
+    int numVectors = 1 + random().nextInt(50);
+    int dim = random().nextInt(64) + 1;
+    if (dim % 2 == 1) {
+      dim++;
+    }
+    VectorSimilarityFunction similarityFunction = randomSimilarity();
+    List<float[]> vectors = getRandomFloatVector(numVectors, dim, false);
+    float[] query = getRandomFloatVector(1, dim, false).get(0);
+
+    try (BaseDirectoryWrapper dir = newDirectory()) {
+      // The raw vector file is emptied below, which CheckIndex rightly rejects.
+      dir.setCheckIndexOnClose(false);
+
+      try (IndexWriter w =
+          new IndexWriter(
+              dir,
+              new IndexWriterConfig()
+                  .setMaxBufferedDocs(numVectors + 1)
+                  .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
+                  .setMergePolicy(NoMergePolicy.INSTANCE)
+                  .setUseCompoundFile(false)
+                  .setCodec(getCodecForFloatVectorFallbackTest()))) {
+        for (int i = 0; i < numVectors; i++) {
+          Document doc = new Document();
+          doc.add(new KnnFloatVectorField(vectorFieldName, vectors.get(i), similarityFunction));
+          w.addDocument(doc);
+        }
+      }
+
+      // Scores while the raw full-precision vectors are still present.
+      Map<Integer, Float> expectedScores = scoreAllDocs(dir, vectorFieldName, query);
+      assertEquals("expected every document to be scored", numVectors, expectedScores.size());
+
+      simulateEmptyRawVectors(dir);
+
+      // Both reads are expected to score against the same quantized vectors, so the scores must
+      // match exactly rather than merely within a quantization-error tolerance.
+      Map<Integer, Float> actualScores = scoreAllDocs(dir, vectorFieldName, query);
+      assertEquals(expectedScores.keySet(), actualScores.keySet());
+      for (Map.Entry<Integer, Float> entry : expectedScores.entrySet()) {
+        assertEquals(
+            "score changed for doc " + entry.getKey() + " after dropping raw vectors",
+            entry.getValue(),
+            actualScores.get(entry.getKey()),
+            0f);
+      }
+    }
+  }
+
+  /**
+   * Scores every document holding a vector for {@code field} against {@code query}, keyed by doc
+   * id. Keying by doc id rather than ordinal keeps the comparison meaningful even when a different
+   * {@link FloatVectorValues} implementation backs the iterator.
+   */
+  private Map<Integer, Float> scoreAllDocs(Directory dir, String field, float[] query)
+      throws IOException {
+    Map<Integer, Float> scores = new HashMap<>();
+    try (IndexReader reader = DirectoryReader.open(dir)) {
+      LeafReader leafReader = getOnlyLeafReader(reader);
+      if (leafReader instanceof CodecReader codecReader) {
+        KnnVectorsReader knnVectorsReader =
+            codecReader.getVectorReader().unwrapReaderForField(field);
+        FloatVectorValues floatVectorValues = knnVectorsReader.getFloatVectorValues(field);
+        assertNotNull(floatVectorValues);
+        VectorScorer scorer = floatVectorValues.scorer(query);
+        assertNotNull(scorer);
+        DocIdSetIterator iterator = scorer.iterator();
+        for (int doc = iterator.nextDoc(); doc != NO_MORE_DOCS; doc = iterator.nextDoc()) {
+          scores.put(doc, scorer.score());
+        }
+      } else {
+        fail("reader is not CodecReader");
+      }
+    }
+    return scores;
+  }
+
   /**
    * Simulates empty raw vectors by modifying index files. Override in codecs that support
    * FloatVector fallback.