You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2025/11/13 16:05:49 UTC
(lucene) branch main updated: Add fallback support to Lucene104ScalarQuantizedVectorsFormat getFloatVectorValues when there are no full-precision vectors present (#15415)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new ba455f2bae1 Add fallback support to Lucene104ScalarQuantizedVectorsFormat getFloatVectorValues when there are no full-precision vectors present (#15415)
ba455f2bae1 is described below
commit ba455f2bae13e39c25474f19a5c292e3c09b7611
Author: Pulkit Gupta <36...@users.noreply.github.com>
AuthorDate: Thu Nov 13 11:05:42 2025 -0500
Add fallback support to Lucene104ScalarQuantizedVectorsFormat getFloatVectorValues when there are no full-precision vectors present (#15415)
* Add fallback support to Lucene104ScalarQuantizedVectorsFormat.getFloatVectorValues when there are no full-precision vectors present
* Removed some unused params and tidy changes
* Added entry in Changes.txt
* Added name in Changes.txt log
* Addressed comments
* Renamed function params
---------
Co-authored-by: Pulkit Gupta <pu...@amazon.com>
---
lucene/CHANGES.txt | 3 +
.../Lucene104ScalarQuantizedVectorsReader.java | 19 +-
.../OffHeapScalarQuantizedFloatVectorValues.java | 389 +++++++++++++++++++++
.../quantization/OptimizedScalarQuantizer.java | 49 +++
.../TestLucene104ScalarQuantizedVectorsFormat.java | 168 +++++++++
.../quantization/TestOptimizedScalarQuantizer.java | 46 ++-
6 files changed, 654 insertions(+), 20 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0742b8d96fa..c53632a7c2e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -176,6 +176,9 @@ New Features
`Lucene104HnswScalarQuantizedVectorsFormat(ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE, int, int)`
(Ben Trent)
+ * GITHUB#15415: Add fallback support to Lucene104ScalarQuantizedVectorsFormat getFloatVectorValues when there are
+ no full-precision vectors present (Pulkit Gupta)
+
Improvements
---------------------
* GITHUB#15148: Add support uint8 distance and allow 8 bit scalar quantization (Trevor McCulloch)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index d870fd1601c..4a108fe55fd 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -210,6 +210,23 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
+ " expected: "
+ VectorEncoding.FLOAT32);
}
+
+ FloatVectorValues rawFloatVectorValues = rawVectorsReader.getFloatVectorValues(field);
+
+ if (rawFloatVectorValues.size() == 0) {
+ return OffHeapScalarQuantizedFloatVectorValues.load(
+ fi.ordToDocDISIReaderConfiguration,
+ fi.dimension,
+ fi.size,
+ fi.scalarEncoding,
+ fi.similarityFunction,
+ vectorScorer,
+ fi.centroid,
+ fi.vectorDataOffset,
+ fi.vectorDataLength,
+ quantizedVectorData);
+ }
+
OffHeapScalarQuantizedVectorValues sqvv =
OffHeapScalarQuantizedVectorValues.load(
fi.ordToDocDISIReaderConfiguration,
@@ -224,7 +241,7 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
fi.vectorDataOffset,
fi.vectorDataLength,
quantizedVectorData);
- return new ScalarQuantizedVectorValues(rawVectorsReader.getFloatVectorValues(field), sqvv);
+ return new ScalarQuantizedVectorValues(rawFloatVectorValues, sqvv);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java
new file mode 100644
index 00000000000..2576d67224b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene104;
+
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.deQuantize;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
+import org.apache.lucene.codecs.lucene95.HasIndexSlice;
+import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.hnsw.RandomVectorScorer;
+import org.apache.lucene.util.packed.DirectMonotonicReader;
+import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+
+/**
+ * Reads quantized vector values from the index input and returns float vector values after
+ * dequantizing them.
+ *
+ * <p>This class provides functionality to read quantized vectors which are stored in the index, and
+ * then dequantize them back to float vectors with some precision loss. The implementation is based
+ * on {@code OffHeapScalarQuantizedVectorValues} with modifications to the {@code vectorValue()}
+ * method to return float vectors after dequantizing the vectors.
+ *
+ * <p>Usage: This class is used for read-only indexes where full-precision float vectors have been
+ * dropped from the index to save storage space. Full-precision vectors can be removed from an index
+ * using a method as implemented in {@code
+ * TestLucene104ScalarQuantizedVectorsFormat.simulateEmptyRawVectors()}.
+ *
+ * @lucene.internal
+ */
+abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValues
+ implements HasIndexSlice {
+
+ final int dimension;
+ final int size;
+ final VectorSimilarityFunction similarityFunction;
+ final FlatVectorsScorer vectorsScorer;
+
+ final IndexInput slice;
+ final float[] vectorValue;
+ final byte[] byteValue;
+ final ByteBuffer byteBuffer;
+ final byte[] unpackedByteVectorValue;
+ final int byteSize;
+ private int lastOrd = -1;
+ final float[] correctiveValues;
+ int quantizedComponentSum;
+ final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding;
+ final float[] centroid;
+
+ OffHeapScalarQuantizedFloatVectorValues(
+ int dimension,
+ int size,
+ float[] centroid,
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+ VectorSimilarityFunction similarityFunction,
+ FlatVectorsScorer vectorsScorer,
+ IndexInput slice) {
+ this.dimension = dimension;
+ this.size = size;
+ this.similarityFunction = similarityFunction;
+ this.vectorsScorer = vectorsScorer;
+ this.slice = slice;
+ this.centroid = centroid;
+ this.correctiveValues = new float[3];
+ this.encoding = encoding;
+ int docPackedLength = encoding.getDocPackedLength(dimension);
+ this.byteSize = docPackedLength + (Float.BYTES * 3) + Integer.BYTES;
+ this.byteBuffer = ByteBuffer.allocate(docPackedLength);
+ this.vectorValue = new float[dimension];
+ this.byteValue = byteBuffer.array();
+ this.unpackedByteVectorValue = new byte[dimension];
+ }
+
+ @Override
+ public int dimension() {
+ return dimension;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public float[] vectorValue(int targetOrd) throws IOException {
+ if (lastOrd == targetOrd) {
+ return vectorValue;
+ }
+
+ // read quantized byte vector, correctiveValues and quantizedComponentSum
+ slice.seek((long) targetOrd * byteSize);
+ slice.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), byteValue.length);
+ slice.readFloats(correctiveValues, 0, 3);
+ quantizedComponentSum = slice.readInt();
+
+ // unpack bytes
+ switch (encoding) {
+ case PACKED_NIBBLE ->
+ OffHeapScalarQuantizedVectorValues.unpackNibbles(byteValue, unpackedByteVectorValue);
+ case SINGLE_BIT_QUERY_NIBBLE ->
+ OptimizedScalarQuantizer.unpackBinary(byteValue, unpackedByteVectorValue);
+ case UNSIGNED_BYTE, SEVEN_BIT -> {
+ deQuantize(
+ byteValue,
+ vectorValue,
+ encoding.getBits(),
+ correctiveValues[0],
+ correctiveValues[1],
+ centroid);
+ lastOrd = targetOrd;
+ return vectorValue;
+ }
+ }
+
+ // dequantize
+ deQuantize(
+ unpackedByteVectorValue,
+ vectorValue,
+ encoding.getBits(),
+ correctiveValues[0],
+ correctiveValues[1],
+ centroid);
+
+ lastOrd = targetOrd;
+ return vectorValue;
+ }
+
+ public OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int targetOrd)
+ throws IOException {
+ if (lastOrd == targetOrd) {
+ return new OptimizedScalarQuantizer.QuantizationResult(
+ correctiveValues[0], correctiveValues[1], correctiveValues[2], quantizedComponentSum);
+ }
+ slice.seek(((long) targetOrd * byteSize) + byteValue.length);
+ slice.readFloats(correctiveValues, 0, 3);
+ quantizedComponentSum = slice.readInt();
+ return new OptimizedScalarQuantizer.QuantizationResult(
+ correctiveValues[0], correctiveValues[1], correctiveValues[2], quantizedComponentSum);
+ }
+
+ @Override
+ public int getVectorByteLength() {
+ return dimension;
+ }
+
+ @Override
+ public IndexInput getSlice() {
+ return slice;
+ }
+
+ static OffHeapScalarQuantizedFloatVectorValues load(
+ OrdToDocDISIReaderConfiguration configuration,
+ int dimension,
+ int size,
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+ VectorSimilarityFunction similarityFunction,
+ FlatVectorsScorer vectorsScorer,
+ float[] centroid,
+ long quantizedVectorDataOffset,
+ long quantizedVectorDataLength,
+ IndexInput vectorData)
+ throws IOException {
+ if (configuration.isEmpty()) {
+ return new OffHeapScalarQuantizedFloatVectorValues.EmptyOffHeapVectorValues(
+ dimension, similarityFunction, vectorsScorer);
+ }
+ assert centroid != null;
+ IndexInput bytesSlice =
+ vectorData.slice(
+ "scalar-quantized-float-vector-data",
+ quantizedVectorDataOffset,
+ quantizedVectorDataLength);
+ if (configuration.isDense()) {
+ return new OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues(
+ dimension, size, centroid, encoding, similarityFunction, vectorsScorer, bytesSlice);
+ } else {
+ return new OffHeapScalarQuantizedFloatVectorValues.SparseOffHeapVectorValues(
+ configuration,
+ dimension,
+ size,
+ centroid,
+ encoding,
+ vectorData,
+ similarityFunction,
+ vectorsScorer,
+ bytesSlice);
+ }
+ }
+
+ /** Dense off-heap scalar quantized vector values */
+ private static class DenseOffHeapVectorValues extends OffHeapScalarQuantizedFloatVectorValues {
+ DenseOffHeapVectorValues(
+ int dimension,
+ int size,
+ float[] centroid,
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+ VectorSimilarityFunction similarityFunction,
+ FlatVectorsScorer vectorsScorer,
+ IndexInput slice) {
+ super(dimension, size, centroid, encoding, similarityFunction, vectorsScorer, slice);
+ }
+
+ @Override
+ public OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues copy()
+ throws IOException {
+ return new OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues(
+ dimension, size, centroid, encoding, similarityFunction, vectorsScorer, slice.clone());
+ }
+
+ @Override
+ public Bits getAcceptOrds(Bits acceptDocs) {
+ return acceptDocs;
+ }
+
+ @Override
+ public VectorScorer scorer(float[] target) throws IOException {
+ OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues copy = copy();
+ DocIndexIterator iterator = copy.iterator();
+ RandomVectorScorer scorer =
+ vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);
+ return new VectorScorer() {
+ @Override
+ public float score() throws IOException {
+ return scorer.score(iterator.index());
+ }
+
+ @Override
+ public DocIdSetIterator iterator() {
+ return iterator;
+ }
+ };
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return createDenseIterator();
+ }
+ }
+
+ /** Sparse off-heap scalar quantized vector values */
+ private static class SparseOffHeapVectorValues extends OffHeapScalarQuantizedFloatVectorValues {
+ private final DirectMonotonicReader ordToDoc;
+ private final IndexedDISI disi;
+ // dataIn was used to init a new IndexedDIS for #randomAccess()
+ private final IndexInput dataIn;
+ private final OrdToDocDISIReaderConfiguration configuration;
+
+ SparseOffHeapVectorValues(
+ OrdToDocDISIReaderConfiguration configuration,
+ int dimension,
+ int size,
+ float[] centroid,
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+ IndexInput dataIn,
+ VectorSimilarityFunction similarityFunction,
+ FlatVectorsScorer vectorsScorer,
+ IndexInput slice)
+ throws IOException {
+ super(dimension, size, centroid, encoding, similarityFunction, vectorsScorer, slice);
+ this.configuration = configuration;
+ this.dataIn = dataIn;
+ this.ordToDoc = configuration.getDirectMonotonicReader(dataIn);
+ this.disi = configuration.getIndexedDISI(dataIn);
+ }
+
+ @Override
+ public OffHeapScalarQuantizedFloatVectorValues.SparseOffHeapVectorValues copy()
+ throws IOException {
+ return new OffHeapScalarQuantizedFloatVectorValues.SparseOffHeapVectorValues(
+ configuration,
+ dimension,
+ size,
+ centroid,
+ encoding,
+ dataIn,
+ similarityFunction,
+ vectorsScorer,
+ slice.clone());
+ }
+
+ @Override
+ public int ordToDoc(int ord) {
+ return (int) ordToDoc.get(ord);
+ }
+
+ @Override
+ public Bits getAcceptOrds(Bits acceptDocs) {
+ if (acceptDocs == null) {
+ return null;
+ }
+ return new Bits() {
+ @Override
+ public boolean get(int index) {
+ return acceptDocs.get(ordToDoc(index));
+ }
+
+ @Override
+ public int length() {
+ return size;
+ }
+ };
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return IndexedDISI.asDocIndexIterator(disi);
+ }
+
+ @Override
+ public VectorScorer scorer(float[] target) throws IOException {
+ OffHeapScalarQuantizedFloatVectorValues.SparseOffHeapVectorValues copy = copy();
+ DocIndexIterator iterator = copy.iterator();
+ RandomVectorScorer scorer =
+ vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);
+ return new VectorScorer() {
+ @Override
+ public float score() throws IOException {
+ return scorer.score(iterator.index());
+ }
+
+ @Override
+ public DocIdSetIterator iterator() {
+ return iterator;
+ }
+ };
+ }
+ }
+
+ /** Empty vector values */
+ private static class EmptyOffHeapVectorValues extends OffHeapScalarQuantizedFloatVectorValues {
+ EmptyOffHeapVectorValues(
+ int dimension,
+ VectorSimilarityFunction similarityFunction,
+ FlatVectorsScorer vectorsScorer) {
+ super(
+ dimension,
+ 0,
+ null,
+ Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.UNSIGNED_BYTE,
+ similarityFunction,
+ vectorsScorer,
+ null);
+ }
+
+ @Override
+ public DocIndexIterator iterator() {
+ return createDenseIterator();
+ }
+
+ @Override
+ public OffHeapScalarQuantizedFloatVectorValues.DenseOffHeapVectorValues copy() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Bits getAcceptOrds(Bits acceptDocs) {
+ return null;
+ }
+
+ @Override
+ public VectorScorer scorer(float[] target) {
+ return null;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java
index b807220e7b2..9f937043a13 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/OptimizedScalarQuantizer.java
@@ -236,6 +236,36 @@ public class OptimizedScalarQuantizer {
sumQuery);
}
+ /**
+ * Dequantizes a quantized byte vector back to float values.
+ *
+ * <p>This method reconstructs float vectors from their quantized byte representation using linear
+ * interpolation between the corrective range [a, b] and adding back the centroid offset.
+ *
+ * @param quantized the quantized byte vector to dequantize
+ * @param dequantized the output array to store dequantized float values
+ * @param bits the number of bits used for quantization
+ * @param lowerInterval lower value of quantization range
+ * @param upperInterval upper value of quantization range
+ * @param centroid the centroid vector that was subtracted during quantization
+ * @return the dequantized float array (same as dequantized parameter)
+ */
+ public static float[] deQuantize(
+ byte[] quantized,
+ float[] dequantized,
+ byte bits,
+ float lowerInterval,
+ float upperInterval,
+ float[] centroid) {
+ int nSteps = (1 << bits) - 1;
+ double step = (upperInterval - lowerInterval) / nSteps;
+ for (int h = 0; h < quantized.length; h++) {
+ double xi = (double) (quantized[h] & 0xFF) * step + lowerInterval;
+ dequantized[h] = (float) (xi + centroid[h]);
+ }
+ return dequantized;
+ }
+
/**
* Compute the loss of the vector given the interval. Effectively, we are computing the MSE of a
* dequantized vector with the raw vector.
@@ -391,6 +421,25 @@ public class OptimizedScalarQuantizer {
}
}
+ /**
+ * Unpack a binary array back to a vector.
+ *
+ * @param packed the packed binary array
+ * @param vector the unpacked vector (each byte will be 0 or 1)
+ */
+ public static void unpackBinary(byte[] packed, byte[] vector) {
+ int vectorIndex = 0;
+ for (int packedIndex = 0;
+ packedIndex < packed.length && vectorIndex < vector.length;
+ packedIndex++) {
+ byte packedByte = packed[packedIndex];
+ for (int j = 7; j >= 0 && vectorIndex < vector.length; j--) {
+ vector[vectorIndex] = (byte) ((packedByte >> j) & 1);
+ vectorIndex++;
+ }
+ }
+ }
+
private static double clamp(double x, double a, double b) {
return Math.min(Math.max(x, a), b);
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
index 150294df12c..32d5b07ad3e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
@@ -17,18 +17,25 @@
package org.apache.lucene.codecs.lucene104;
import static java.lang.String.format;
+import static org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.oneOf;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Locale;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
+import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexReader;
@@ -36,6 +43,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnFloatVectorQuery;
@@ -43,8 +51,13 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
+import org.apache.lucene.tests.store.BaseDirectoryWrapper;
import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
import org.junit.Before;
@@ -204,4 +217,159 @@ public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsFor
}
}
}
+
+ protected List<float[]> getRandomFloatVector(int numVectors, int dim, boolean normalize) {
+ List<float[]> vectors = new ArrayList<>(numVectors);
+ for (int i = 0; i < numVectors; i++) {
+ float[] vec = randomVector(dim);
+ if (normalize) {
+ float[] copy = new float[vec.length];
+ System.arraycopy(vec, 0, copy, 0, copy.length);
+ VectorUtil.l2normalize(copy);
+ vec = copy;
+ }
+ vectors.add(vec);
+ }
+ return vectors;
+ }
+
+ public void testReadQuantizedVectorWithEmptyRawVectors() throws Exception {
+ String vectorFieldName = "vec1";
+ int numVectors = 1 + random().nextInt(50);
+ int dim = random().nextInt(64) + 1;
+ if (dim % 2 == 1) {
+ dim++;
+ }
+ float eps = (1f / (float) (1 << (encoding.getBits())));
+ VectorSimilarityFunction similarityFunction = randomSimilarity();
+ List<float[]> vectors =
+ getRandomFloatVector(
+ numVectors, dim, similarityFunction == VectorSimilarityFunction.COSINE);
+
+ try (BaseDirectoryWrapper dir = newDirectory();
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig()
+ .setMaxBufferedDocs(numVectors + 1)
+ .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
+ .setMergePolicy(NoMergePolicy.INSTANCE)
+ .setUseCompoundFile(false)
+ .setCodec(getCodec()))) {
+ dir.setCheckIndexOnClose(false);
+
+ for (int i = 0; i < numVectors; i++) {
+ Document doc = new Document();
+ doc.add(new KnnFloatVectorField(vectorFieldName, vectors.get(i), similarityFunction));
+ w.addDocument(doc);
+ }
+ w.commit();
+
+ simulateEmptyRawVectors(dir);
+
+ try (IndexReader reader = DirectoryReader.open(w)) {
+ LeafReader r = getOnlyLeafReader(reader);
+ if (r instanceof CodecReader codecReader) {
+ KnnVectorsReader knnVectorsReader = codecReader.getVectorReader();
+ knnVectorsReader = knnVectorsReader.unwrapReaderForField(vectorFieldName);
+ if (knnVectorsReader instanceof Lucene104ScalarQuantizedVectorsReader quantizedReader) {
+ FloatVectorValues floatVectorValues =
+ quantizedReader.getFloatVectorValues(vectorFieldName);
+ if (floatVectorValues instanceof OffHeapScalarQuantizedFloatVectorValues) {
+ KnnVectorValues.DocIndexIterator iter = floatVectorValues.iterator();
+ for (int docId = iter.nextDoc(); docId != NO_MORE_DOCS; docId = iter.nextDoc()) {
+ float[] dequantizedVector = floatVectorValues.vectorValue(iter.index());
+ float mae = 0;
+ for (int i = 0; i < dim; i++) {
+ mae += Math.abs(dequantizedVector[i] - vectors.get(docId)[i]);
+ }
+ mae /= dim;
+ assertTrue(
+ "bits: " + encoding.getBits() + " mae: " + mae + " > eps: " + eps, mae <= eps);
+ }
+ } else {
+ fail("floatVectorValues is not OffHeapScalarQuantizedFloatVectorValues");
+ }
+ } else {
+ System.out.println("Vector READER:: " + knnVectorsReader.toString());
+ fail("reader is not Lucene104ScalarQuantizedVectorsReader");
+ }
+ } else {
+ fail("reader is not CodecReader");
+ }
+ }
+ }
+ }
+
+ /** Simulates empty raw vectors by modifying index files. */
+ private void simulateEmptyRawVectors(Directory dir) throws Exception {
+ final String[] indexFiles = dir.listAll();
+ final String RAW_VECTOR_EXTENSION = "vec";
+ final String VECTOR_META_EXTENSION = "vemf";
+
+ for (String file : indexFiles) {
+ if (file.endsWith("." + RAW_VECTOR_EXTENSION)) {
+ replaceWithEmptyVectorFile(dir, file);
+ } else if (file.endsWith("." + VECTOR_META_EXTENSION)) {
+ updateVectorMetadataFile(dir, file);
+ }
+ }
+ }
+
+ /** Replaces a raw vector file with an empty one that has valid header/footer. */
+ private void replaceWithEmptyVectorFile(Directory dir, String fileName) throws Exception {
+ byte[] indexHeader;
+ try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
+ indexHeader = CodecUtil.readIndexHeader(in);
+ }
+ dir.deleteFile(fileName);
+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+ // Write header
+ out.writeBytes(indexHeader, 0, indexHeader.length);
+ // Write footer (no content in between)
+ CodecUtil.writeFooter(out);
+ }
+ }
+
+ /** Updates vector metadata file to indicate zero vector length. */
+ private void updateVectorMetadataFile(Directory dir, String fileName) throws Exception {
+ // Read original metadata
+ byte[] indexHeader;
+ int fieldNumber, vectorEncoding, vectorSimilarityFunction, dimension;
+ long vectorStartPos;
+
+ try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
+ indexHeader = CodecUtil.readIndexHeader(in);
+ fieldNumber = in.readInt();
+ vectorEncoding = in.readInt();
+ vectorSimilarityFunction = in.readInt();
+ vectorStartPos = in.readVLong();
+ in.readVLong(); // Skip original vector length
+ dimension = in.readVInt();
+ }
+
+ // Create updated metadata file
+ dir.deleteFile(fileName);
+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+ // Write header
+ out.writeBytes(indexHeader, 0, indexHeader.length);
+
+ // Write metadata with zero vector length
+ out.writeInt(fieldNumber);
+ out.writeInt(vectorEncoding);
+ out.writeInt(vectorSimilarityFunction);
+ out.writeVLong(vectorStartPos);
+ out.writeVLong(0); // Set vector length to 0
+ out.writeVInt(dimension);
+ out.writeInt(0);
+
+ // Write configuration
+ OrdToDocDISIReaderConfiguration.writeStoredMeta(
+ DIRECT_MONOTONIC_BLOCK_SHIFT, out, null, 0, 0, null);
+
+ // Mark end of fields and write footer
+ out.writeInt(-1);
+ CodecUtil.writeFooter(out);
+ }
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java b/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
index 7f3684648d6..62bd242531d 100644
--- a/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
+++ b/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
@@ -16,10 +16,15 @@
*/
package org.apache.lucene.util.quantization;
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomFloat;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;
import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.MINIMUM_MSE_GRID;
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.deQuantize;
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.packAsBinary;
+import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.unpackBinary;
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.VectorUtil;
@@ -27,19 +32,6 @@ import org.apache.lucene.util.VectorUtil;
public class TestOptimizedScalarQuantizer extends LuceneTestCase {
static final byte[] ALL_BITS = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
- static float[] deQuantize(byte[] quantized, byte bits, float[] interval, float[] centroid) {
- float[] dequantized = new float[quantized.length];
- float a = interval[0];
- float b = interval[1];
- int nSteps = (1 << bits) - 1;
- double step = (b - a) / nSteps;
- for (int h = 0; h < quantized.length; h++) {
- double xi = (double) (quantized[h] & 0xFF) * step + a;
- dequantized[h] = (float) (xi + centroid[h]);
- }
- return dequantized;
- }
-
public void testQuantizationQuality() {
int dims = 16;
int numVectors = 32;
@@ -69,12 +61,14 @@ public class TestOptimizedScalarQuantizer extends LuceneTestCase {
assertValidResults(result);
assertValidQuantizedRange(destination, bit);
- float[] dequantized =
- deQuantize(
- destination,
- bit,
- new float[] {result.lowerInterval(), result.upperInterval()},
- centroid);
+ float[] dequantized = new float[dims];
+ deQuantize(
+ destination,
+ dequantized,
+ bit,
+ result.lowerInterval(),
+ result.upperInterval(),
+ centroid);
float mae = 0;
for (int k = 0; k < dims; ++k) {
mae += Math.abs(dequantized[k] - vectors[i][k]);
@@ -186,6 +180,20 @@ public class TestOptimizedScalarQuantizer extends LuceneTestCase {
}
}
+ public void testUnpackBinary() {
+ int dim = randomIntBetween(1, 4096);
+ ScalarEncoding encoding = ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE;
+ byte[] scratch = new byte[encoding.getDiscreteDimensions(dim)];
+ for (int i = 0; i < scratch.length; i++) {
+ scratch[i] = randomBoolean() ? (byte) 1 : (byte) 0;
+ }
+ byte[] packed = new byte[encoding.getDocPackedLength(scratch.length)];
+ byte[] unpacked = new byte[scratch.length];
+ packAsBinary(scratch, packed);
+ unpackBinary(packed, unpacked);
+ assertArrayEquals(scratch, unpacked);
+ }
+
static void assertValidQuantizedRange(byte[] quantized, byte bits) {
for (byte b : quantized) {
if (bits < 8) {