You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2025/11/13 16:05:49 UTC

(lucene) branch main updated: Add fallback support to Lucene104ScalarQuantizedVectorsFormat getFloatVectorValues when there are no full-precision vectors present (#15415)

This is an automated email from the ASF dual-hosted git repository.

benwtrent pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new ba455f2bae1 Add fallback support to Lucene104ScalarQuantizedVectorsFormat getFloatVectorValues when there are no full-precision vectors present (#15415)
ba455f2bae1 is described below

commit ba455f2bae13e39c25474f19a5c292e3c09b7611
Author: Pulkit Gupta <36...@users.noreply.github.com>
AuthorDate: Thu Nov 13 11:05:42 2025 -0500

    Add fallback support to Lucene104ScalarQuantizedVectorsFormat getFloatVectorValues when there are no full-precision vectors present (#15415)
    
    * Add fallback support to Lucene104ScalarQuantizedVectorsFormat.getFloatVectorValues when there are no full-precision vectors present
    
    * Removed some unused params and tidy changes
    
    * Added entry in Changes.txt
    
    * Added name in Changes.txt log
    
    * Addressed comments
    
    * Renamed function params
    
    ---------
    
    Co-authored-by: Pulkit Gupta <pu...@amazon.com>
---
 lucene/CHANGES.txt                                 |   3 +
 .../Lucene104ScalarQuantizedVectorsReader.java     |  19 +-
 .../OffHeapScalarQuantizedFloatVectorValues.java   | 389 +++++++++++++++++++++
 .../quantization/OptimizedScalarQuantizer.java     |  49 +++
 .../TestLucene104ScalarQuantizedVectorsFormat.java | 168 +++++++++
 .../quantization/TestOptimizedScalarQuantizer.java |  46 ++-
 6 files changed, 654 insertions(+), 20 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0742b8d96fa..c53632a7c2e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -176,6 +176,9 @@ New Features
    `Lucene104HnswScalarQuantizedVectorsFormat(ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE, int, int)`
     (Ben Trent)
 
+ * GITHUB#15415: Add fallback support to Lucene104ScalarQuantizedVectorsFormat getFloatVectorValues when there are
+   no full-precision vectors present (Pulkit Gupta)
+
 Improvements
 ---------------------
 * GITHUB#15148: Add support uint8 distance and allow 8 bit scalar quantization (Trevor McCulloch)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index d870fd1601c..4a108fe55fd 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -210,6 +210,23 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
               + " expected: "
               + VectorEncoding.FLOAT32);
     }
+
+    FloatVectorValues rawFloatVectorValues = rawVectorsReader.getFloatVectorValues(field);
+
+    if (rawFloatVectorValues.size() == 0) {
+      return OffHeapScalarQuantizedFloatVectorValues.load(
+          fi.ordToDocDISIReaderConfiguration,
+          fi.dimension,
+          fi.size,
+          fi.scalarEncoding,
+          fi.similarityFunction,
+          vectorScorer,
+          fi.centroid,
+          fi.vectorDataOffset,
+          fi.vectorDataLength,
+          quantizedVectorData);
+    }
+
     OffHeapScalarQuantizedVectorValues sqvv =
         OffHeapScalarQuantizedVectorValues.load(
             fi.ordToDocDISIReaderConfiguration,
@@ -224,7 +241,7 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
             fi.vectorDataOffset,
             fi.vectorDataLength,
             quantizedVectorData);
-    return new ScalarQuantizedVectorValues(rawVectorsReader.getFloatVectorValues(field), sqvv);
+    return new ScalarQuantizedVectorValues(rawFloatVectorValues, sqvv);
   }
 
   @Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java
new file mode 100644
index 00000000000..2576d67224b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene104;
+
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.deQuantize;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
+import org.apache.lucene.codecs.lucene95.HasIndexSlice;
+import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.packed.DirectMonotonicReader;
+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+
+/**
+ * Reads quantized vector values from the index input and returns float vector values after
+ * dequantizing them.
+ *
+ * <p>This class provides functionality to read quantized vectors which are stored in the index, and
+ * then dequantize them back to float vectors with some precision loss. The implementation is based
+ * on {@code OffHeapScalarQuantizedVectorValues} with modifications to the {@code vectorValue()}
+ * method to return float vectors after dequantizing the vectors.
+ *
+ * <p>Usage: This class is used for read-only indexes where full-precision float vectors have been
+ * dropped from the index to save storage space. Full-precision vectors can be removed from an index
+ * using a method as implemented in {@code
+ * TestLucene104ScalarQuantizedVectorsFormat.simulateEmptyRawVectors()}.
+ *
+ * @lucene.internal
+ */
+abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValues
+    implements HasIndexSlice {
+
+  final int dimension;
+  final int size;
+  final VectorSimilarityFunction similarityFunction;
+  final FlatVectorsScorer vectorsScorer;
+
+  final IndexInput slice;
+  final float[] vectorValue;
+  final byte[] byteValue;
+  final ByteBuffer byteBuffer;
+  final byte[] unpackedByteVectorValue;
+  final int byteSize;
+  private int lastOrd = -1;
+  final float[] correctiveValues;
+  int quantizedComponentSum;
+  final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding;
+  final float[] centroid;
+
+  OffHeapScalarQuantizedFloatVectorValues(
+      int dimension,
+      int size,
+      float[] centroid,
+      Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+      VectorSimilarityFunction similarityFunction,
+      FlatVectorsScorer vectorsScorer,
+      IndexInput slice) {
+    this.dimension = dimension;
+    this.size = size;
+    this.similarityFunction = similarityFunction;
+    this.vectorsScorer = vectorsScorer;
+    this.slice = slice;
+    this.centroid = centroid;
+    this.correctiveValues = new float[3];
+    this.encoding = encoding;
+    int docPackedLength = encoding.getDocPackedLength(dimension);
+    this.byteSize = docPackedLength + (Float.BYTES * 3) + Integer.BYTES;
+    this.byteBuffer = ByteBuffer.allocate(docPackedLength);
+    this.vectorValue = new float[dimension];
+    this.byteValue = byteBuffer.array();
+    this.unpackedByteVectorValue = new byte[dimension];
+  }
+
+  @Override
+  public int dimension() {
+    return dimension;
+  }
+
+  @Override
+  public int size() {
+    return size;
+  }
+
+  @Override
+  public float[] vectorValue(int targetOrd) throws IOException {
+    if (lastOrd == targetOrd) {
+      return vectorValue;
+    }
+
+    // read quantized byte vector, correctiveValues and quantizedComponentSum
+    slice.seek((long) targetOrd * byteSize);
+    slice.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), byteValue.length);
+    slice.readFloats(correctiveValues, 0, 3);
+    quantizedComponentSum = slice.readInt();
+
+    // unpack bytes
+    switch (encoding) {
+      case PACKED_NIBBLE ->
+          OffHeapScalarQuantizedVectorValues.unpackNibbles(byteValue, unpackedByteVectorValue);
+      case SINGLE_BIT_QUERY_NIBBLE ->
+          OptimizedScalarQuantizer.unpackBinary(byteValue, unpackedByteVectorValue);
+      case UNSIGNED_BYTE, SEVEN_BIT -> {
+        deQuantize(
+            byteValue,
+            vectorValue,
+            encoding.getBits(),
+            correctiveValues[0],
+            correctiveValues[1],
+            centroid);
+        lastOrd = targetOrd;
+        return vectorValue;
+      }
+    }
+
+    // dequantize
+    deQuantize(
+        unpackedByteVectorValue,
+        vectorValue,
+        encoding.getBits(),
+        correctiveValues[0],
+        correctiveValues[1],
+        centroid);
+
+    lastOrd = targetOrd;
+    return vectorValue;
+  }
+
+  public OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int targetOrd)
+      throws IOException {
+    if (lastOrd == targetOrd) {
+      return new OptimizedScalarQuantizer.QuantizationResult(
+          correctiveValues[0], correctiveValues[1], correctiveValues[2], quantizedComponentSum);
+    }
+    slice.seek(((long) targetOrd * byteSize) + byteValue.length);
+    slice.readFloats(correctiveValues, 0, 3);
+    quantizedComponentSum = slice.readInt();
+    return new OptimizedScalarQuantizer.QuantizationResult(
+        correctiveValues[0], correctiveValues[1], correctiveValues[2], quantizedComponentSum);
+  }
+
+  @Override
+  public int getVectorByteLength() {
+    return dimension;
+  }
+
+  @Override
+  public IndexInput getSlice() {
+    return slice;
+  }
+
+  static OffHeapScalarQuantizedFloatVectorValues load(
+      OrdToDocDISIReaderConfiguration configuration,
+      int dimension,
+      int size,
+      Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+      VectorSimilarityFunction similarityFunction,
+      FlatVectorsScorer vectorsScorer,
+      float[] centroid,
+      long quantizedVectorDataOffset,
+      long quantizedVectorDataLength,
+      IndexInput vectorData)
+      throws IOException {
+    if (configuration.isEmpty()) {
+      return new OffHeapScalarQuantizedFloatVectorValues.EmptyOffHeapVectorValues(
+          dimension, similarityFunction, vectorsScorer);
+    }
+    assert centroid != null;
+    IndexInput bytesSlice =
+        vectorData.slice(
+            "scalar-quantized-float-vector-data",
+            quantizedVectorDataOffset,
+            quantizedVectorDataLength);
+    if (configuration.isDense()) {
+      return new OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues(
+          dimension, size, centroid, encoding, similarityFunction, vectorsScorer, bytesSlice);
+    } else {
+      return new OffHeapScalarQuantizedFloatVectorValues.SparseOffHeapVectorValues(
+          configuration,
+          dimension,
+          size,
+          centroid,
+          encoding,
+          vectorData,
+          similarityFunction,
+          vectorsScorer,
+          bytesSlice);
+    }
+  }
+
+  /** Dense off-heap scalar quantized vector values */
+  private static class DenseOffHeapVectorValues extends OffHeapScalarQuantizedFloatVectorValues {
+    DenseOffHeapVectorValues(
+        int dimension,
+        int size,
+        float[] centroid,
+        Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+        VectorSimilarityFunction similarityFunction,
+        FlatVectorsScorer vectorsScorer,
+        IndexInput slice) {
+      super(dimension, size, centroid, encoding, similarityFunction, vectorsScorer, slice);
+    }
+
+    @Override
+    public OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues copy()
+        throws IOException {
+      return new OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues(
+          dimension, size, centroid, encoding, similarityFunction, vectorsScorer, slice.clone());
+    }
+
+    @Override
+    public Bits getAcceptOrds(Bits acceptDocs) {
+      return acceptDocs;
+    }
+
+    @Override
+    public VectorScorer scorer(float[] target) throws IOException {
+      OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues copy = copy();
+      DocIndexIterator iterator = copy.iterator();
+      RandomVectorScorer scorer =
+          vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);
+      return new VectorScorer() {
+        @Override
+        public float score() throws IOException {
+          return scorer.score(iterator.index());
+        }
+
+        @Override
+        public DocIdSetIterator iterator() {
+          return iterator;
+        }
+      };
+    }
+
+    @Override
+    public DocIndexIterator iterator() {
+      return createDenseIterator();
+    }
+  }
+
+  /** Sparse off-heap scalar quantized vector values */
+  private static class SparseOffHeapVectorValues extends OffHeapScalarQuantizedFloatVectorValues {
+    private final DirectMonotonicReader ordToDoc;
+    private final IndexedDISI disi;
+    // dataIn was used to init a new IndexedDIS for #randomAccess()
+    private final IndexInput dataIn;
+    private final OrdToDocDISIReaderConfiguration configuration;
+
+    SparseOffHeapVectorValues(
+        OrdToDocDISIReaderConfiguration configuration,
+        int dimension,
+        int size,
+        float[] centroid,
+        Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+        IndexInput dataIn,
+        VectorSimilarityFunction similarityFunction,
+        FlatVectorsScorer vectorsScorer,
+        IndexInput slice)
+        throws IOException {
+      super(dimension, size, centroid, encoding, similarityFunction, vectorsScorer, slice);
+      this.configuration = configuration;
+      this.dataIn = dataIn;
+      this.ordToDoc = configuration.getDirectMonotonicReader(dataIn);
+      this.disi = configuration.getIndexedDISI(dataIn);
+    }
+
+    @Override
+    public OffHeapScalarQuantizedFloatVectorValues.SparseOffHeapVectorValues copy()
+        throws IOException {
+      return new OffHeapScalarQuantizedFloatVectorValues.SparseOffHeapVectorValues(
+          configuration,
+          dimension,
+          size,
+          centroid,
+          encoding,
+          dataIn,
+          similarityFunction,
+          vectorsScorer,
+          slice.clone());
+    }
+
+    @Override
+    public int ordToDoc(int ord) {
+      return (int) ordToDoc.get(ord);
+    }
+
+    @Override
+    public Bits getAcceptOrds(Bits acceptDocs) {
+      if (acceptDocs == null) {
+        return null;
+      }
+      return new Bits() {
+        @Override
+        public boolean get(int index) {
+          return acceptDocs.get(ordToDoc(index));
+        }
+
+        @Override
+        public int length() {
+          return size;
+        }
+      };
+    }
+
+    @Override
+    public DocIndexIterator iterator() {
+      return IndexedDISI.asDocIndexIterator(disi);
+    }
+
+    @Override
+    public VectorScorer scorer(float[] target) throws IOException {
+      OffHeapScalarQuantizedFloatVectorValues.SparseOffHeapVectorValues copy = copy();
+      DocIndexIterator iterator = copy.iterator();
+      RandomVectorScorer scorer =
+          vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);
+      return new VectorScorer() {
+        @Override
+        public float score() throws IOException {
+          return scorer.score(iterator.index());
+        }
+
+        @Override
+        public DocIdSetIterator iterator() {
+          return iterator;
+        }
+      };
+    }
+  }
+
+  /** Empty vector values */
+  private static class EmptyOffHeapVectorValues extends OffHeapScalarQuantizedFloatVectorValues {
+    EmptyOffHeapVectorValues(
+        int dimension,
+        VectorSimilarityFunction similarityFunction,
+        FlatVectorsScorer vectorsScorer) {
+      super(
+          dimension,
+          0,
+          null,
+          Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.UNSIGNED_BYTE,
+          similarityFunction,
+          vectorsScorer,
+          null);
+    }
+
+    @Override
+    public DocIndexIterator iterator() {
+      return createDenseIterator();
+    }
+
+    @Override
+    public OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues copy() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Bits getAcceptOrds(Bits acceptDocs) {
+      return null;
+    }
+
+    @Override
+    public VectorScorer scorer(float[] target) {
+      return null;
+    }
+  }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java
index b807220e7b2..9f937043a13 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java
@@ -236,6 +236,36 @@ public class OptimizedScalarQuantizer {
         sumQuery);
   }
 
+  /**
+   * Dequantizes a quantized byte vector back to float values.
+   *
+   * <p>This method reconstructs float vectors from their quantized byte representation using linear
+   * interpolation between the corrective range [a, b] and adding back the centroid offset.
+   *
+   * @param quantized the quantized byte vector to dequantize
+   * @param dequantized the output array to store dequantized float values
+   * @param bits the number of bits used for quantization
+   * @param lowerInterval lower value of quantization range
+   * @param upperInterval upper value of quantization range
+   * @param centroid the centroid vector that was subtracted during quantization
+   * @return the dequantized float array (same as dequantized parameter)
+   */
+  public static float[] deQuantize(
+      byte[] quantized,
+      float[] dequantized,
+      byte bits,
+      float lowerInterval,
+      float upperInterval,
+      float[] centroid) {
+    int nSteps = (1 << bits) - 1;
+    double step = (upperInterval - lowerInterval) / nSteps;
+    for (int h = 0; h < quantized.length; h++) {
+      double xi = (double) (quantized[h] & 0xFF) * step + lowerInterval;
+      dequantized[h] = (float) (xi + centroid[h]);
+    }
+    return dequantized;
+  }
+
   /**
    * Compute the loss of the vector given the interval. Effectively, we are computing the MSE of a
    * dequantized vector with the raw vector.
@@ -391,6 +421,25 @@ public class OptimizedScalarQuantizer {
     }
   }
 
+  /**
+   * Unpack a binary array back to a vector.
+   *
+   * @param packed the packed binary array
+   * @param vector the unpacked vector (each byte will be 0 or 1)
+   */
+  public static void unpackBinary(byte[] packed, byte[] vector) {
+    int vectorIndex = 0;
+    for (int packedIndex = 0;
+        packedIndex < packed.length && vectorIndex < vector.length;
+        packedIndex++) {
+      byte packedByte = packed[packedIndex];
+      for (int j = 7; j >= 0 && vectorIndex < vector.length; j--) {
+        vector[vectorIndex] = (byte) ((packedByte >> j) & 1);
+        vectorIndex++;
+      }
+    }
+  }
+
   private static double clamp(double x, double a, double b) {
     return Math.min(Math.max(x, a), b);
   }
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
index 150294df12c..32d5b07ad3e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
@@ -17,18 +17,25 @@
 package org.apache.lucene.codecs.lucene104;
 
 import static java.lang.String.format;
+import static org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.oneOf;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Locale;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
 import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
+import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.index.CodecReader;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexReader;
@@ -36,6 +43,7 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.KnnVectorValues;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.KnnFloatVectorQuery;
@@ -43,8 +51,13 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
+import org.apache.lucene.tests.store.BaseDirectoryWrapper;
 import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.junit.Before;
 
@@ -204,4 +217,159 @@ public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsFor
       }
     }
   }
+
+  protected List<float[]> getRandomFloatVector(int numVectors, int dim, boolean normalize) {
+    List<float[]> vectors = new ArrayList<>(numVectors);
+    for (int i = 0; i < numVectors; i++) {
+      float[] vec = randomVector(dim);
+      if (normalize) {
+        float[] copy = new float[vec.length];
+        System.arraycopy(vec, 0, copy, 0, copy.length);
+        VectorUtil.l2normalize(copy);
+        vec = copy;
+      }
+      vectors.add(vec);
+    }
+    return vectors;
+  }
+
+  public void testReadQuantizedVectorWithEmptyRawVectors() throws Exception {
+    String vectorFieldName = "vec1";
+    int numVectors = 1 + random().nextInt(50);
+    int dim = random().nextInt(64) + 1;
+    if (dim % 2 == 1) {
+      dim++;
+    }
+    float eps = (1f / (float) (1 << (encoding.getBits())));
+    VectorSimilarityFunction similarityFunction = randomSimilarity();
+    List<float[]> vectors =
+        getRandomFloatVector(
+            numVectors, dim, similarityFunction == VectorSimilarityFunction.COSINE);
+
+    try (BaseDirectoryWrapper dir = newDirectory();
+        IndexWriter w =
+            new IndexWriter(
+                dir,
+                new IndexWriterConfig()
+                    .setMaxBufferedDocs(numVectors + 1)
+                    .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
+                    .setMergePolicy(NoMergePolicy.INSTANCE)
+                    .setUseCompoundFile(false)
+                    .setCodec(getCodec()))) {
+      dir.setCheckIndexOnClose(false);
+
+      for (int i = 0; i < numVectors; i++) {
+        Document doc = new Document();
+        doc.add(new KnnFloatVectorField(vectorFieldName, vectors.get(i), similarityFunction));
+        w.addDocument(doc);
+      }
+      w.commit();
+
+      simulateEmptyRawVectors(dir);
+
+      try (IndexReader reader = DirectoryReader.open(w)) {
+        LeafReader r = getOnlyLeafReader(reader);
+        if (r instanceof CodecReader codecReader) {
+          KnnVectorsReader knnVectorsReader = codecReader.getVectorReader();
+          knnVectorsReader = knnVectorsReader.unwrapReaderForField(vectorFieldName);
+          if (knnVectorsReader instanceof Lucene104ScalarQuantizedVectorsReader quantizedReader) {
+            FloatVectorValues floatVectorValues =
+                quantizedReader.getFloatVectorValues(vectorFieldName);
+            if (floatVectorValues instanceof OffHeapScalarQuantizedFloatVectorValues) {
+              KnnVectorValues.DocIndexIterator iter = floatVectorValues.iterator();
+              for (int docId = iter.nextDoc(); docId != NO_MORE_DOCS; docId = iter.nextDoc()) {
+                float[] dequantizedVector = floatVectorValues.vectorValue(iter.index());
+                float mae = 0;
+                for (int i = 0; i < dim; i++) {
+                  mae += Math.abs(dequantizedVector[i] - vectors.get(docId)[i]);
+                }
+                mae /= dim;
+                assertTrue(
+                    "bits: " + encoding.getBits() + " mae: " + mae + " > eps: " + eps, mae <= eps);
+              }
+            } else {
+              fail("floatVectorValues is not OffHeapScalarQuantizedFloatVectorValues");
+            }
+          } else {
+            System.out.println("Vector READER:: " + knnVectorsReader.toString());
+            fail("reader is not Lucene104ScalarQuantizedVectorsReader");
+          }
+        } else {
+          fail("reader is not CodecReader");
+        }
+      }
+    }
+  }
+
+  /** Simulates empty raw vectors by modifying index files. */
+  private void simulateEmptyRawVectors(Directory dir) throws Exception {
+    final String[] indexFiles = dir.listAll();
+    final String RAW_VECTOR_EXTENSION = "vec";
+    final String VECTOR_META_EXTENSION = "vemf";
+
+    for (String file : indexFiles) {
+      if (file.endsWith("." + RAW_VECTOR_EXTENSION)) {
+        replaceWithEmptyVectorFile(dir, file);
+      } else if (file.endsWith("." + VECTOR_META_EXTENSION)) {
+        updateVectorMetadataFile(dir, file);
+      }
+    }
+  }
+
+  /** Replaces a raw vector file with an empty one that has valid header/footer. */
+  private void replaceWithEmptyVectorFile(Directory dir, String fileName) throws Exception {
+    byte[] indexHeader;
+    try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
+      indexHeader = CodecUtil.readIndexHeader(in);
+    }
+    dir.deleteFile(fileName);
+    try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+      // Write header
+      out.writeBytes(indexHeader, 0, indexHeader.length);
+      // Write footer (no content in between)
+      CodecUtil.writeFooter(out);
+    }
+  }
+
+  /** Updates vector metadata file to indicate zero vector length. */
+  private void updateVectorMetadataFile(Directory dir, String fileName) throws Exception {
+    // Read original metadata
+    byte[] indexHeader;
+    int fieldNumber, vectorEncoding, vectorSimilarityFunction, dimension;
+    long vectorStartPos;
+
+    try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
+      indexHeader = CodecUtil.readIndexHeader(in);
+      fieldNumber = in.readInt();
+      vectorEncoding = in.readInt();
+      vectorSimilarityFunction = in.readInt();
+      vectorStartPos = in.readVLong();
+      in.readVLong(); // Skip original vector length
+      dimension = in.readVInt();
+    }
+
+    // Create updated metadata file
+    dir.deleteFile(fileName);
+    try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+      // Write header
+      out.writeBytes(indexHeader, 0, indexHeader.length);
+
+      // Write metadata with zero vector length
+      out.writeInt(fieldNumber);
+      out.writeInt(vectorEncoding);
+      out.writeInt(vectorSimilarityFunction);
+      out.writeVLong(vectorStartPos);
+      out.writeVLong(0); // Set vector length to 0
+      out.writeVInt(dimension);
+      out.writeInt(0);
+
+      // Write configuration
+      OrdToDocDISIReaderConfiguration.writeStoredMeta(
+          DIRECT_MONOTONIC_BLOCK_SHIFT, out, null, 0, 0, null);
+
+      // Mark end of fields and write footer
+      out.writeInt(-1);
+      CodecUtil.writeFooter(out);
+    }
+  }
 }
diff --git a/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java b/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
index 7f3684648d6..62bd242531d 100644
--- a/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
+++ b/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
@@ -16,10 +16,15 @@
  */
 package org.apache.lucene.util.quantization;
 
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
 import static com.carrotsearch.randomizedtesting.RandomizedTest.randomFloat;
 import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;
 import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.MINIMUM_MSE_GRID;
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.deQuantize;
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.packAsBinary;
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.unpackBinary;
 
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.VectorUtil;
@@ -27,19 +32,6 @@ import org.apache.lucene.util.VectorUtil;
 public class TestOptimizedScalarQuantizer extends LuceneTestCase {
   static final byte[] ALL_BITS = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
 
-  static float[] deQuantize(byte[] quantized, byte bits, float[] interval, float[] centroid) {
-    float[] dequantized = new float[quantized.length];
-    float a = interval[0];
-    float b = interval[1];
-    int nSteps = (1 << bits) - 1;
-    double step = (b - a) / nSteps;
-    for (int h = 0; h < quantized.length; h++) {
-      double xi = (double) (quantized[h] & 0xFF) * step + a;
-      dequantized[h] = (float) (xi + centroid[h]);
-    }
-    return dequantized;
-  }
-
   public void testQuantizationQuality() {
     int dims = 16;
     int numVectors = 32;
@@ -69,12 +61,14 @@ public class TestOptimizedScalarQuantizer extends LuceneTestCase {
         assertValidResults(result);
         assertValidQuantizedRange(destination, bit);
 
-        float[] dequantized =
-            deQuantize(
-                destination,
-                bit,
-                new float[] {result.lowerInterval(), result.upperInterval()},
-                centroid);
+        float[] dequantized = new float[dims];
+        deQuantize(
+            destination,
+            dequantized,
+            bit,
+            result.lowerInterval(),
+            result.upperInterval(),
+            centroid);
         float mae = 0;
         for (int k = 0; k < dims; ++k) {
           mae += Math.abs(dequantized[k] - vectors[i][k]);
@@ -186,6 +180,20 @@ public class TestOptimizedScalarQuantizer extends LuceneTestCase {
     }
   }
 
+  public void testUnpackBinary() {
+    int dim = randomIntBetween(1, 4096);
+    ScalarEncoding encoding = ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE;
+    byte[] scratch = new byte[encoding.getDiscreteDimensions(dim)];
+    for (int i = 0; i < scratch.length; i++) {
+      scratch[i] = randomBoolean() ? (byte) 1 : (byte) 0;
+    }
+    byte[] packed = new byte[encoding.getDocPackedLength(scratch.length)];
+    byte[] unpacked = new byte[scratch.length];
+    packAsBinary(scratch, packed);
+    unpackBinary(packed, unpacked);
+    assertArrayEquals(scratch, unpacked);
+  }
+
   static void assertValidQuantizedRange(byte[] quantized, byte bits) {
     for (byte b : quantized) {
       if (bits < 8) {