You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by iv...@apache.org on 2026/03/02 15:57:38 UTC
(lucene) branch branch_10x updated: Refactor QuantizedVectorsReader to be able to access lucene 104 quantized values through this API. (#15783)
This is an automated email from the ASF dual-hosted git repository.
ivera pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 0205df24156 Refactor QuantizedVectorsReader to be able to access lucene 104 quantized values through this API. (#15783)
0205df24156 is described below
commit 0205df241562aca280be9481f8612b4ff37b954c
Author: Ignacio Vera <iv...@apache.org>
AuthorDate: Mon Mar 2 15:37:48 2026 +0000
Refactor QuantizedVectorsReader to be able to access lucene 104 quantized values through this API. (#15783)
# Conflicts:
# lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.java
# lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java
---
lucene/CHANGES.txt | 3 +
.../Lucene99ScalarQuantizedVectorsReader.java | 8 +-
.../lucene99/OffHeapQuantizedByteVectorValues.java | 4 +-
.../Lucene99ScalarQuantizedVectorsWriter.java | 25 ++-
.../TestLucene99HnswQuantizedVectorsFormat.java | 6 +-
.../TestLucene99ScalarQuantizedVectorScorer.java | 8 +-
.../TestLucene99ScalarQuantizedVectorsFormat.java | 4 +-
.../TestInt7HnswBackwardsCompatibility.java | 4 +-
.../codecs/hnsw/ScalarQuantizedVectorScorer.java | 14 +-
.../Lucene104HnswScalarQuantizedVectorsFormat.java | 2 +-
.../Lucene104ScalarQuantizedVectorScorer.java | 10 +-
.../Lucene104ScalarQuantizedVectorsFormat.java | 159 +--------------
.../Lucene104ScalarQuantizedVectorsReader.java | 54 ++---
.../Lucene104ScalarQuantizedVectorsWriter.java | 3 +-
.../OffHeapScalarQuantizedFloatVectorValues.java | 13 +-
.../OffHeapScalarQuantizedVectorValues.java | 2 +-
.../lucene104/QuantizedByteVectorValues.java | 96 ---------
.../codecs/lucene99/Lucene99HnswVectorsReader.java | 4 +-
.../Lucene99ScalarQuantizedVectorScorer.java | 34 ++--
...ues.java => BaseQuantizedByteVectorValues.java} | 17 +-
...r.java => LegacyQuantizedByteVectorValues.java} | 18 +-
.../quantization/QuantizedByteVectorValues.java | 225 ++++++++++++++++++---
.../util/quantization/QuantizedVectorsReader.java | 2 +-
.../Lucene99MemorySegmentFlatVectorsScorer.java | 6 +-
...99MemorySegmentScalarQuantizedVectorScorer.java | 14 +-
...tLucene104HnswScalarQuantizedVectorsFormat.java | 2 +-
.../TestLucene104ScalarQuantizedVectorsFormat.java | 3 +-
...PrecisionFloatVectorSimilarityValuesSource.java | 4 +-
.../quantization/TestOptimizedScalarQuantizer.java | 7 +-
29 files changed, 330 insertions(+), 421 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b97861511a1..944aa502015 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -65,6 +65,9 @@ Other
* GITHUB#15767: Fix random test failures in LegacyBaseDocValuesFormatTestCase caused by missing commit. (Alan Woodward)
+* GITHUB#15783: Refactor QuantizedVectorsReader to be able to access lucene 104 quantized values through this API.
+ (Ignacio Vera)
+
======================= Lucene 10.4.0 =======================
API Changes
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index 274afa30f1a..b8fa9cd7667 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -48,7 +48,7 @@ import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.QuantizedVectorsReader;
import org.apache.lucene.util.quantization.ScalarQuantizer;
@@ -333,7 +333,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
}
@Override
- public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {
+ public LegacyQuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {
final FieldEntry fieldEntry = getFieldEntry(field);
return OffHeapQuantizedByteVectorValues.load(
fieldEntry.ordToDoc,
@@ -428,10 +428,10 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
private static final class QuantizedVectorValues extends FloatVectorValues {
private final FloatVectorValues rawVectorValues;
- private final QuantizedByteVectorValues quantizedVectorValues;
+ private final LegacyQuantizedByteVectorValues quantizedVectorValues;
QuantizedVectorValues(
- FloatVectorValues rawVectorValues, QuantizedByteVectorValues quantizedVectorValues) {
+ FloatVectorValues rawVectorValues, LegacyQuantizedByteVectorValues quantizedVectorValues) {
this.rawVectorValues = rawVectorValues;
this.quantizedVectorValues = quantizedVectorValues;
}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java
index 22fe9f2c569..c64ea69d9e5 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/OffHeapQuantizedByteVectorValues.java
@@ -29,14 +29,14 @@ import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.packed.DirectMonotonicReader;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.ScalarQuantizer;
/**
* Read the quantized vector values and their score correction values from the index input. This
* supports both iterated and random access.
*/
-public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVectorValues {
+public abstract class OffHeapQuantizedByteVectorValues extends LegacyQuantizedByteVectorValues {
final int dimension;
final int size;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
index 0deac21e06a..38c28f081e0 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
@@ -62,7 +62,8 @@ import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.QuantizedVectorsReader;
import org.apache.lucene.util.quantization.ScalarQuantizer;
@@ -720,7 +721,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
*/
public static DocsWithFieldSet writeQuantizedVectorData(
IndexOutput output,
- QuantizedByteVectorValues quantizedByteVectorValues,
+ LegacyQuantizedByteVectorValues quantizedByteVectorValues,
byte bits,
boolean compress)
throws IOException {
@@ -890,10 +891,10 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
}
static class QuantizedByteVectorValueSub extends DocIDMerger.Sub {
- private final QuantizedByteVectorValues values;
+ private final LegacyQuantizedByteVectorValues values;
private final KnnVectorValues.DocIndexIterator iterator;
- QuantizedByteVectorValueSub(MergeState.DocMap docMap, QuantizedByteVectorValues values) {
+ QuantizedByteVectorValueSub(MergeState.DocMap docMap, LegacyQuantizedByteVectorValues values) {
super(docMap);
this.values = values;
iterator = values.iterator();
@@ -910,8 +911,8 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
}
}
- /** Returns a merged view over all the segment's {@link QuantizedByteVectorValues}. */
- static class MergedQuantizedVectorValues extends QuantizedByteVectorValues {
+ /** Returns a merged view over all the segment's {@link BaseQuantizedByteVectorValues}. */
+ static class MergedQuantizedVectorValues extends LegacyQuantizedByteVectorValues {
public static MergedQuantizedVectorValues mergeQuantizedByteVectorValues(
FieldInfo fieldInfo, MergeState mergeState, ScalarQuantizer scalarQuantizer)
throws IOException {
@@ -948,7 +949,8 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
new QuantizedByteVectorValueSub(
mergeState.docMaps[i],
new OffsetCorrectedQuantizedByteVectorValues(
- reader.getQuantizedVectorValues(fieldInfo.name),
+ (LegacyQuantizedByteVectorValues)
+ reader.getQuantizedVectorValues(fieldInfo.name),
fieldInfo.getVectorSimilarityFunction(),
scalarQuantizer,
reader.getQuantizationState(fieldInfo.name)));
@@ -1045,7 +1047,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
}
}
- static class QuantizedFloatVectorValues extends QuantizedByteVectorValues {
+ static class QuantizedFloatVectorValues extends LegacyQuantizedByteVectorValues {
private final FloatVectorValues values;
private final ScalarQuantizer quantizer;
private final byte[] quantizedVector;
@@ -1150,14 +1152,15 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
}
}
- static final class OffsetCorrectedQuantizedByteVectorValues extends QuantizedByteVectorValues {
+ static final class OffsetCorrectedQuantizedByteVectorValues
+ extends LegacyQuantizedByteVectorValues {
- private final QuantizedByteVectorValues in;
+ private final LegacyQuantizedByteVectorValues in;
private final VectorSimilarityFunction vectorSimilarityFunction;
private final ScalarQuantizer scalarQuantizer, oldScalarQuantizer;
OffsetCorrectedQuantizedByteVectorValues(
- QuantizedByteVectorValues in,
+ LegacyQuantizedByteVectorValues in,
VectorSimilarityFunction vectorSimilarityFunction,
ScalarQuantizer scalarQuantizer,
ScalarQuantizer oldScalarQuantizer) {
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
index e22e6a958c2..93c5f926518 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
@@ -53,7 +53,7 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.SameThreadExecutorService;
import org.apache.lucene.util.VectorUtil;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.ScalarQuantizer;
import org.junit.Before;
@@ -280,8 +280,8 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormat
}
if (knnVectorsReader instanceof Lucene99HnswVectorsReader hnswReader) {
assertNotNull(hnswReader.getQuantizationState("f"));
- QuantizedByteVectorValues quantizedByteVectorValues =
- hnswReader.getQuantizedVectorValues("f");
+ LegacyQuantizedByteVectorValues quantizedByteVectorValues =
+ (LegacyQuantizedByteVectorValues) hnswReader.getQuantizedVectorValues("f");
for (int ord = 0; ord < quantizedByteVectorValues.size(); ord++) {
byte[] vector = quantizedByteVectorValues.vectorValue(ord);
float offset = quantizedByteVectorValues.getScoreCorrectionConstant(ord);
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
index 71fbff929ab..5d8433119c6 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
@@ -49,7 +49,7 @@ import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.ScalarQuantizer;
public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {
@@ -99,8 +99,8 @@ public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {
try (IndexInput in = dir.openInput(fileName, IOContext.DEFAULT)) {
Lucene99ScalarQuantizedVectorScorer scorer =
new Lucene99ScalarQuantizedVectorScorer(new DefaultFlatVectorScorer());
- QuantizedByteVectorValues values =
- new QuantizedByteVectorValues() {
+ LegacyQuantizedByteVectorValues values =
+ new LegacyQuantizedByteVectorValues() {
@Override
public int dimension() {
return 32;
@@ -127,7 +127,7 @@ public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {
}
@Override
- public QuantizedByteVectorValues copy() throws IOException {
+ public LegacyQuantizedByteVectorValues copy() throws IOException {
return this;
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
index 4387b24b15f..5c7ec159dc2 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
@@ -44,7 +44,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.VectorUtil;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.ScalarQuantizer;
import org.junit.Before;
@@ -172,7 +172,7 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsForm
}
if (knnVectorsReader instanceof Lucene99ScalarQuantizedVectorsReader quantizedReader) {
assertNotNull(quantizedReader.getQuantizationState("f"));
- QuantizedByteVectorValues quantizedByteVectorValues =
+ LegacyQuantizedByteVectorValues quantizedByteVectorValues =
quantizedReader.getQuantizedVectorValues("f");
int docId = -1;
KnnVectorValues.DocIndexIterator iter = quantizedByteVectorValues.iterator();
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
index 4986c87e3e0..a91e2847b01 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
@@ -44,7 +44,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Version;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;
public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTestBase {
@@ -161,7 +161,7 @@ public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTe
assertTrue(forField instanceof Lucene99HnswVectorsReader);
- QuantizedByteVectorValues quantized =
+ BaseQuantizedByteVectorValues quantized =
((Lucene99HnswVectorsReader) forField).getQuantizedVectorValues(KNN_VECTOR_FIELD);
assertNotNull(
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.java
index ad4bb7e3c6d..aa8a1aac45b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/ScalarQuantizedVectorScorer.java
@@ -25,7 +25,7 @@ import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.ScalarQuantizedVectorSimilarity;
import org.apache.lucene.util.quantization.ScalarQuantizer;
@@ -63,7 +63,7 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {
public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)
throws IOException {
- if (vectorValues instanceof QuantizedByteVectorValues quantizedByteVectorValues) {
+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantizedByteVectorValues) {
return new ScalarQuantizedRandomVectorScorerSupplier(
similarityFunction,
quantizedByteVectorValues.getScalarQuantizer(),
@@ -77,7 +77,7 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {
public RandomVectorScorer getRandomVectorScorer(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)
throws IOException {
- if (vectorValues instanceof QuantizedByteVectorValues quantizedByteVectorValues) {
+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantizedByteVectorValues) {
ScalarQuantizer scalarQuantizer = quantizedByteVectorValues.getScalarQuantizer();
byte[] targetBytes = new byte[target.length];
float offsetCorrection =
@@ -121,14 +121,14 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {
public static class ScalarQuantizedRandomVectorScorerSupplier
implements RandomVectorScorerSupplier {
- private final QuantizedByteVectorValues values;
+ private final LegacyQuantizedByteVectorValues values;
private final ScalarQuantizedVectorSimilarity similarity;
private final VectorSimilarityFunction vectorSimilarityFunction;
public ScalarQuantizedRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction,
ScalarQuantizer scalarQuantizer,
- QuantizedByteVectorValues values) {
+ LegacyQuantizedByteVectorValues values) {
this.similarity =
ScalarQuantizedVectorSimilarity.fromVectorSimilarity(
similarityFunction,
@@ -141,7 +141,7 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {
private ScalarQuantizedRandomVectorScorerSupplier(
ScalarQuantizedVectorSimilarity similarity,
VectorSimilarityFunction vectorSimilarityFunction,
- QuantizedByteVectorValues values) {
+ LegacyQuantizedByteVectorValues values) {
this.similarity = similarity;
this.values = values;
this.vectorSimilarityFunction = vectorSimilarityFunction;
@@ -149,7 +149,7 @@ public class ScalarQuantizedVectorScorer implements FlatVectorsScorer {
@Override
public UpdateableRandomVectorScorer scorer() throws IOException {
- final QuantizedByteVectorValues vectorsCopy = values.copy();
+ final LegacyQuantizedByteVectorValues vectorsCopy = values.copy();
byte[] queryVector = new byte[values.dimension()];
return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(vectorsCopy) {
float queryOffset = 0;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
index 0bb271436d1..4fd3dd6dbcf 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
@@ -28,7 +28,6 @@ import java.util.concurrent.ExecutorService;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.KnnVectorsWriter;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;
@@ -36,6 +35,7 @@ import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.TaskExecutor;
import org.apache.lucene.util.hnsw.HnswGraph;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
/**
* A vectors format that uses HNSW graph to store and search for vectors. But vectors are binary
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
index 69deea50e15..53d3363e64b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorScorer.java
@@ -30,6 +30,8 @@ import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
/**
* Vector scorer over OptimizedScalarQuantized vectors
@@ -61,7 +63,7 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
if (vectorValues instanceof QuantizedByteVectorValues qv) {
FlatVectorsScorer.checkDimensions(target.length, qv.dimension());
OptimizedScalarQuantizer quantizer = qv.getQuantizer();
- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding scalarEncoding = qv.getScalarEncoding();
+ ScalarEncoding scalarEncoding = qv.getScalarEncoding();
byte[] scratch = new byte[scalarEncoding.getDiscreteDimensions(qv.dimension())];
final byte[] targetQuantized;
if (scalarEncoding.isAsymmetric() == false) {
@@ -81,10 +83,8 @@ public class Lucene104ScalarQuantizedVectorScorer implements FlatVectorsScorer {
target, scratch, scalarEncoding.getQueryBits(), qv.getCentroid());
// for asymmetric encodings with 4-bit query, we need to transpose the nibbles for fast
// scoring comparisons
- if (scalarEncoding
- == Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE
- || scalarEncoding
- == Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.DIBIT_QUERY_NIBBLE) {
+ if (scalarEncoding == ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE
+ || scalarEncoding == ScalarEncoding.DIBIT_QUERY_NIBBLE) {
OptimizedScalarQuantizer.transposeHalfByte(scratch, targetQuantized);
}
return new RandomVectorScorer.AbstractRandomVectorScorer(qv) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
index a5a1c1de97c..fb8221deffb 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
@@ -17,7 +17,6 @@
package org.apache.lucene.codecs.lucene104;
import java.io.IOException;
-import java.util.Optional;
import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
@@ -25,6 +24,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
/**
* The quantization format used here is a per-vector optimized scalar quantization. These ideas are
@@ -112,163 +112,6 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
private final ScalarEncoding encoding;
- /**
- * Allowed encodings for scalar quantization.
- *
- * <p>This specifies how many bits are used per dimension and also dictates packing of dimensions
- * into a byte stream.
- */
- public enum ScalarEncoding {
- /** Each dimension is quantized to 8 bits and treated as an unsigned value. */
- UNSIGNED_BYTE(0, (byte) 8, 8),
- /** Each dimension is quantized to 4 bits two values are packed into each output byte. */
- PACKED_NIBBLE(1, (byte) 4, 4),
- /**
- * Each dimension is quantized to 7 bits and treated as a signed value.
- *
- * <p>This is intended for backwards compatibility with older iterations of scalar quantization.
- * This setting will produce an index the same size as {@link #UNSIGNED_BYTE} but will produce
- * less accurate vector comparisons.
- */
- SEVEN_BIT(2, (byte) 7, 8),
- /**
- * Each dimension is quantized to a single bit and packed into bytes. During query time, the
- * query vector is quantized to 4 bits per dimension.
- *
- * <p>This is the most space efficient encoding, and will produce an index 8x smaller than
- * {@link #UNSIGNED_BYTE}. However, this comes at the cost of accuracy.
- */
- SINGLE_BIT_QUERY_NIBBLE(3, (byte) 1, 1, (byte) 4, 4),
- /**
- * Each dimension is quantized to 2 bits (dibit) and packed into bytes. During query time, the
- * query vector is quantized to 4 bits per dimension.
- *
- * <p>This encoding produces an index 4x smaller than {@link #UNSIGNED_BYTE}, offering a balance
- * between the compression of {@link #SINGLE_BIT_QUERY_NIBBLE} and the accuracy of {@link
- * #PACKED_NIBBLE}.
- */
- DIBIT_QUERY_NIBBLE(4, (byte) 2, 2, (byte) 4, 4) {
- @Override
- public int getDiscreteDimensions(int dimensions) {
- int queryDiscretized = (dimensions * 4 + 7) / 8 * 8 / 4;
- // we want to force dibit packing to byte boundaries assuming single bit striping
- // so we discretize to the same as single bit encoding
- int docDiscretized = (dimensions + 7) / 8 * 8;
- int maxDiscretized = Math.max(queryDiscretized, docDiscretized);
- assert maxDiscretized % (8.0 / 4) == 0
- : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
- assert maxDiscretized % (8.0 / 2) == 0
- : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
- return maxDiscretized;
- }
-
- @Override
- public int getDocPackedLength(int dimensions) {
- int discretized = getDiscreteDimensions(dimensions);
- // DIBIT should be stored as two single bits striped
- return 2 * ((discretized + 7) / 8);
- }
- };
-
- public static ScalarEncoding fromNumBits(int bits) {
- for (ScalarEncoding encoding : values()) {
- if (encoding.bits == bits) {
- return encoding;
- }
- }
- throw new IllegalArgumentException("No encoding for " + bits + " bits");
- }
-
- /** The number used to identify this encoding on the wire, rather than relying on ordinal. */
- private final int wireNumber;
-
- private final byte bits, queryBits;
- private final int bitsPerDim, queryBitsPerDim;
-
- ScalarEncoding(int wireNumber, byte bits, int bitsPerDim) {
- this.wireNumber = wireNumber;
- this.bits = bits;
- this.queryBits = bits;
- this.bitsPerDim = bitsPerDim;
- this.queryBitsPerDim = bitsPerDim;
- }
-
- ScalarEncoding(int wireNumber, byte bits, int bitsPerDim, byte queryBits, int queryBitsPerDim) {
- this.wireNumber = wireNumber;
- this.bits = bits;
- this.queryBits = queryBits;
- this.bitsPerDim = bitsPerDim;
- this.queryBitsPerDim = queryBitsPerDim;
- }
-
- public boolean isAsymmetric() {
- return bits != queryBits;
- }
-
- int getWireNumber() {
- return wireNumber;
- }
-
- /** Return the number of bits used per dimension. */
- public byte getBits() {
- return bits;
- }
-
- public byte getQueryBits() {
- return queryBits;
- }
-
- /** Return the number of dimensions rounded up to fit into whole bytes. */
- public int getDiscreteDimensions(int dimensions) {
- if (queryBits == bits) {
- int totalBits = dimensions * bitsPerDim;
- return (totalBits + 7) / 8 * 8 / bitsPerDim;
- }
- int queryDiscretized = (dimensions * queryBitsPerDim + 7) / 8 * 8 / queryBitsPerDim;
- int docDiscretized = (dimensions * bitsPerDim + 7) / 8 * 8 / bitsPerDim;
- int maxDiscretized = Math.max(queryDiscretized, docDiscretized);
- assert maxDiscretized % (8.0 / queryBitsPerDim) == 0
- : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
- assert maxDiscretized % (8.0 / bitsPerDim) == 0
- : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
- return maxDiscretized;
- }
-
- /** Return the number of dimensions that can be packed into a single byte. */
- public int getDocBitsPerDim() {
- return this.bitsPerDim;
- }
-
- public int getQueryBitsPerDim() {
- return this.queryBitsPerDim;
- }
-
- /** Return the number of bytes required to store a packed vector of the given dimensions. */
- public int getDocPackedLength(int dimensions) {
- int discretized = getDiscreteDimensions(dimensions);
- // how many bytes do we need to store the quantized vector?
- int totalBits = discretized * bitsPerDim;
- return (totalBits + 7) / 8;
- }
-
- public int getQueryPackedLength(int dimensions) {
- int discretized = getDiscreteDimensions(dimensions);
- // how many bytes do we need to store the quantized vector?
- int totalBits = discretized * queryBitsPerDim;
- return (totalBits + 7) / 8;
- }
-
- /** Returns the encoding for the given wire number, or empty if unknown. */
- public static Optional<ScalarEncoding> fromWireNumber(int wireNumber) {
- for (ScalarEncoding encoding : values()) {
- if (encoding.wireNumber == wireNumber) {
- return Optional.of(encoding);
- }
- }
- return Optional.empty();
- }
- }
-
/** Creates a new instance with UNSIGNED_BYTE encoding. */
public Lucene104ScalarQuantizedVectorsFormat() {
this(ScalarEncoding.UNSIGNED_BYTE);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index 65d76fba487..c0c1b97eb0e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -27,7 +27,6 @@ import java.util.Objects;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.CorruptIndexException;
@@ -52,6 +51,8 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
import org.apache.lucene.util.quantization.QuantizedVectorsReader;
import org.apache.lucene.util.quantization.ScalarQuantizer;
@@ -389,8 +390,7 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
}
@Override
- public org.apache.lucene.util.quantization.QuantizedByteVectorValues getQuantizedVectorValues(
- String field) throws IOException {
+ public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {
FieldEntry fi = fields.get(field);
if (fi == null) {
return null;
@@ -404,41 +404,19 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
+ " expected: "
+ VectorEncoding.FLOAT32);
}
- var qv =
- OffHeapScalarQuantizedVectorValues.load(
- fi.ordToDocDISIReaderConfiguration,
- fi.dimension,
- fi.size,
- new OptimizedScalarQuantizer(fi.similarityFunction),
- fi.scalarEncoding,
- fi.similarityFunction,
- vectorScorer,
- fi.centroid,
- fi.centroidDP,
- fi.vectorDataOffset,
- fi.vectorDataLength,
- quantizedVectorData);
- return new org.apache.lucene.util.quantization.QuantizedByteVectorValues() {
- @Override
- public float getScoreCorrectionConstant(int ord) throws IOException {
- return 0;
- }
-
- @Override
- public byte[] vectorValue(int ord) throws IOException {
- return qv.vectorValue(ord);
- }
-
- @Override
- public int dimension() {
- return qv.dimension();
- }
-
- @Override
- public int size() {
- return qv.size();
- }
- };
+ return OffHeapScalarQuantizedVectorValues.load(
+ fi.ordToDocDISIReaderConfiguration,
+ fi.dimension,
+ fi.size,
+ new OptimizedScalarQuantizer(fi.similarityFunction),
+ fi.scalarEncoding,
+ fi.similarityFunction,
+ vectorScorer,
+ fi.centroid,
+ fi.centroidDP,
+ fi.vectorDataOffset,
+ fi.vectorDataLength,
+ quantizedVectorData);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
index 8e0ac0133dc..6187de13fdd 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
@@ -36,7 +36,6 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter;
import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.index.DocsWithFieldSet;
@@ -60,6 +59,8 @@ import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
/**
* Writes quantized vector values and metadata to index segments in the format for Lucene 10.4.
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java
index ec3dbbea087..8fa87a271d2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedFloatVectorValues.java
@@ -34,6 +34,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.packed.DirectMonotonicReader;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
/**
* Reads quantized vector values from the index input and returns float vector values after
@@ -68,14 +69,14 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValues
private int lastOrd = -1;
final float[] correctiveValues;
int quantizedComponentSum;
- final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding;
+ final ScalarEncoding encoding;
final float[] centroid;
OffHeapScalarQuantizedFloatVectorValues(
int dimension,
int size,
float[] centroid,
- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+ ScalarEncoding encoding,
VectorSimilarityFunction similarityFunction,
FlatVectorsScorer vectorsScorer,
IndexInput slice) {
@@ -178,7 +179,7 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValues
OrdToDocDISIReaderConfiguration configuration,
int dimension,
int size,
- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+ ScalarEncoding encoding,
VectorSimilarityFunction similarityFunction,
FlatVectorsScorer vectorsScorer,
float[] centroid,
@@ -219,7 +220,7 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValues
int dimension,
int size,
float[] centroid,
- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+ ScalarEncoding encoding,
VectorSimilarityFunction similarityFunction,
FlatVectorsScorer vectorsScorer,
IndexInput slice) {
@@ -281,7 +282,7 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValues
int dimension,
int size,
float[] centroid,
- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding,
+ ScalarEncoding encoding,
IndexInput dataIn,
VectorSimilarityFunction similarityFunction,
FlatVectorsScorer vectorsScorer,
@@ -372,7 +373,7 @@ abstract class OffHeapScalarQuantizedFloatVectorValues extends FloatVectorValues
dimension,
0,
null,
- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.UNSIGNED_BYTE,
+ ScalarEncoding.UNSIGNED_BYTE,
similarityFunction,
vectorsScorer,
null);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
index 9cfbf8acc6b..a91ace8ff9f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/OffHeapScalarQuantizedVectorValues.java
@@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene104;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
import org.apache.lucene.codecs.lucene90.IndexedDISI;
import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
import org.apache.lucene.index.VectorSimilarityFunction;
@@ -30,6 +29,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.packed.DirectMonotonicReader;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
/**
* Scalar quantized vector values loaded from off-heap
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java
deleted file mode 100644
index 72f35bfc3c7..00000000000
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/QuantizedByteVectorValues.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs.lucene104;
-
-import java.io.IOException;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
-import org.apache.lucene.codecs.lucene95.HasIndexSlice;
-import org.apache.lucene.index.ByteVectorValues;
-import org.apache.lucene.search.VectorScorer;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
-
-/**
- * Scalar quantized byte vector values
- *
- * @lucene.experimental
- */
-public abstract class QuantizedByteVectorValues extends ByteVectorValues implements HasIndexSlice {
-
- /**
- * Retrieve the corrective terms for the given vector ordinal. For the dot-product family of
- * distances, the corrective terms are, in order
- *
- * <ul>
- * <li>the lower optimized interval
- * <li>the upper optimized interval
- * <li>the dot-product of the non-centered vector with the centroid
- * <li>the sum of quantized components
- * </ul>
- *
- * For euclidean:
- *
- * <ul>
- * <li>the lower optimized interval
- * <li>the upper optimized interval
- * <li>the l2norm of the centered vector
- * <li>the sum of quantized components
- * </ul>
- *
- * @param vectorOrd the vector ordinal
- * @return the corrective terms
- * @throws IOException if an I/O error occurs
- */
- public abstract OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int vectorOrd)
- throws IOException;
-
- /**
- * @return the quantizer used to quantize the vectors
- */
- public abstract OptimizedScalarQuantizer getQuantizer();
-
- /**
- * @return the scalar encoding used to pack the stored vectors.
- */
- public abstract ScalarEncoding getScalarEncoding();
-
- /**
- * @return the centroid used to center the vectors prior to quantization
- */
- public abstract float[] getCentroid() throws IOException;
-
- /**
- * @return the dot product of the centroid.
- */
- public abstract float getCentroidDP() throws IOException;
-
- /**
- * Return a {@link VectorScorer} for the given query vector.
- *
- * @param query the query vector
- * @return a {@link VectorScorer} instance or null
- */
- public abstract VectorScorer scorer(float[] query) throws IOException;
-
- @Override
- public abstract QuantizedByteVectorValues copy() throws IOException;
-
- @Override
- public IndexInput getSlice() {
- return null;
- }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
index 6bb0812cb0d..251b92a91da 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
@@ -59,7 +59,7 @@ import org.apache.lucene.util.hnsw.HnswGraphSearcher;
import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.packed.DirectMonotonicReader;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.QuantizedVectorsReader;
import org.apache.lucene.util.quantization.ScalarQuantizer;
@@ -425,7 +425,7 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader
}
@Override
- public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {
+ public BaseQuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException {
if (flatVectorsReader instanceof QuantizedVectorsReader) {
return ((QuantizedVectorsReader) flatVectorsReader).getQuantizedVectorValues(field);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java
index a668c01de6b..57f5b77f290 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java
@@ -28,7 +28,8 @@ import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.ScalarQuantizer;
/**
@@ -49,7 +50,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)
throws IOException {
- if (vectorValues instanceof QuantizedByteVectorValues quantizedByteVectorValues) {
+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantizedByteVectorValues) {
return new ScalarQuantizedRandomVectorScorerSupplier(
quantizedByteVectorValues, similarityFunction);
}
@@ -61,7 +62,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
public RandomVectorScorer getRandomVectorScorer(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)
throws IOException {
- if (vectorValues instanceof QuantizedByteVectorValues quantizedByteVectorValues) {
+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantizedByteVectorValues) {
ScalarQuantizer scalarQuantizer = quantizedByteVectorValues.getScalarQuantizer();
byte[] targetBytes = new byte[target.length];
float offsetCorrection =
@@ -94,7 +95,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
float offsetCorrection,
VectorSimilarityFunction sim,
float constMultiplier,
- QuantizedByteVectorValues values) {
+ LegacyQuantizedByteVectorValues values) {
FlatVectorsScorer.checkDimensions(values.dimension(), targetBytes.length);
return switch (sim) {
case EUCLIDEAN -> new Euclidean(values, constMultiplier, targetBytes);
@@ -120,7 +121,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
byte[] targetBytes,
float offsetCorrection,
float constMultiplier,
- QuantizedByteVectorValues values,
+ LegacyQuantizedByteVectorValues values,
FloatToFloatFunction scoreAdjustmentFunction) {
if (values.getScalarQuantizer().getBits() <= 4) {
if (values.getVectorByteLength() != values.dimension() && values.getSlice() != null) {
@@ -138,9 +139,10 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
extends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {
private final float constMultiplier;
private final byte[] targetBytes;
- private final QuantizedByteVectorValues values;
+ private final BaseQuantizedByteVectorValues values;
- private Euclidean(QuantizedByteVectorValues values, float constMultiplier, byte[] targetBytes) {
+ private Euclidean(
+ BaseQuantizedByteVectorValues values, float constMultiplier, byte[] targetBytes) {
super(values);
this.values = values;
this.constMultiplier = constMultiplier;
@@ -165,13 +167,13 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
private static class DotProduct
extends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {
private final float constMultiplier;
- private final QuantizedByteVectorValues values;
+ private final LegacyQuantizedByteVectorValues values;
private final byte[] targetBytes;
private float offsetCorrection;
private final FloatToFloatFunction scoreAdjustmentFunction;
public DotProduct(
- QuantizedByteVectorValues values,
+ LegacyQuantizedByteVectorValues values,
float constMultiplier,
byte[] targetBytes,
float offsetCorrection,
@@ -210,14 +212,14 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
private static class CompressedInt4DotProduct
extends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {
private final float constMultiplier;
- private final QuantizedByteVectorValues values;
+ private final LegacyQuantizedByteVectorValues values;
private final byte[] compressedVector;
private final byte[] targetBytes;
private float offsetCorrection;
private final FloatToFloatFunction scoreAdjustmentFunction;
private CompressedInt4DotProduct(
- QuantizedByteVectorValues values,
+ LegacyQuantizedByteVectorValues values,
float constMultiplier,
byte[] targetBytes,
float offsetCorrection,
@@ -257,13 +259,13 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
private static class Int4DotProduct
extends UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer {
private final float constMultiplier;
- private final QuantizedByteVectorValues values;
+ private final LegacyQuantizedByteVectorValues values;
private final byte[] targetBytes;
private float offsetCorrection;
private final FloatToFloatFunction scoreAdjustmentFunction;
public Int4DotProduct(
- QuantizedByteVectorValues values,
+ LegacyQuantizedByteVectorValues values,
float constMultiplier,
byte[] targetBytes,
float offsetCorrection,
@@ -299,11 +301,11 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
implements RandomVectorScorerSupplier {
private final VectorSimilarityFunction vectorSimilarityFunction;
- private final QuantizedByteVectorValues values;
- private final QuantizedByteVectorValues targetVectors;
+ private final LegacyQuantizedByteVectorValues values;
+ private final LegacyQuantizedByteVectorValues targetVectors;
public ScalarQuantizedRandomVectorScorerSupplier(
- QuantizedByteVectorValues values, VectorSimilarityFunction vectorSimilarityFunction)
+ LegacyQuantizedByteVectorValues values, VectorSimilarityFunction vectorSimilarityFunction)
throws IOException {
this.values = values;
this.targetVectors = values.copy();
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/util/quantization/BaseQuantizedByteVectorValues.java
similarity index 74%
copy from lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java
copy to lucene/core/src/java/org/apache/lucene/util/quantization/BaseQuantizedByteVectorValues.java
index b90ab8276dd..5e18fbaee1e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/BaseQuantizedByteVectorValues.java
@@ -23,18 +23,12 @@ import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.IndexInput;
/**
- * A version of {@link ByteVectorValues}, but additionally retrieving score correction offset for
- * Scalar quantization scores.
+ * A version of {@link ByteVectorValues} for Scalar quantization scores.
*
* @lucene.experimental
*/
-public abstract class QuantizedByteVectorValues extends ByteVectorValues implements HasIndexSlice {
-
- public ScalarQuantizer getScalarQuantizer() {
- throw new UnsupportedOperationException();
- }
-
- public abstract float getScoreCorrectionConstant(int ord) throws IOException;
+public abstract class BaseQuantizedByteVectorValues extends ByteVectorValues
+ implements HasIndexSlice {
/**
* Return a {@link VectorScorer} for the given query vector.
@@ -46,11 +40,6 @@ public abstract class QuantizedByteVectorValues extends ByteVectorValues impleme
throw new UnsupportedOperationException();
}
- @Override
- public QuantizedByteVectorValues copy() throws IOException {
- return this;
- }
-
@Override
public IndexInput getSlice() {
return null;
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/util/quantization/LegacyQuantizedByteVectorValues.java
similarity index 63%
copy from lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java
copy to lucene/core/src/java/org/apache/lucene/util/quantization/LegacyQuantizedByteVectorValues.java
index 7c49b1f10d8..9b472f8afc0 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/LegacyQuantizedByteVectorValues.java
@@ -16,18 +16,24 @@
*/
package org.apache.lucene.util.quantization;
-import java.io.Closeable;
import java.io.IOException;
-import org.apache.lucene.util.Accountable;
/**
- * Quantized vector reader
+ * A version of {@link BaseQuantizedByteVectorValues}, but additionally retrieving score correction
+ * offset for Scalar quantization scores.
*
* @lucene.experimental
*/
-public interface QuantizedVectorsReader extends Closeable, Accountable {
+public abstract class LegacyQuantizedByteVectorValues extends BaseQuantizedByteVectorValues {
- QuantizedByteVectorValues getQuantizedVectorValues(String fieldName) throws IOException;
+ public ScalarQuantizer getScalarQuantizer() {
+ throw new UnsupportedOperationException();
+ }
- ScalarQuantizer getQuantizationState(String fieldName);
+ public abstract float getScoreCorrectionConstant(int ord) throws IOException;
+
+ @Override
+ public LegacyQuantizedByteVectorValues copy() throws IOException {
+ return this;
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java
index b90ab8276dd..25f47ca38fa 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java
@@ -17,42 +17,219 @@
package org.apache.lucene.util.quantization;
import java.io.IOException;
-import org.apache.lucene.codecs.lucene95.HasIndexSlice;
-import org.apache.lucene.index.ByteVectorValues;
-import org.apache.lucene.search.VectorScorer;
-import org.apache.lucene.store.IndexInput;
+import java.util.Optional;
/**
- * A version of {@link ByteVectorValues}, but additionally retrieving score correction offset for
- * Scalar quantization scores.
+ * Scalar quantized byte vector values
*
* @lucene.experimental
*/
-public abstract class QuantizedByteVectorValues extends ByteVectorValues implements HasIndexSlice {
+public abstract class QuantizedByteVectorValues extends BaseQuantizedByteVectorValues {
- public ScalarQuantizer getScalarQuantizer() {
- throw new UnsupportedOperationException();
- }
+ /**
+ * Allowed encodings for scalar quantization.
+ *
+ * <p>This specifies how many bits are used per dimension and also dictates packing of dimensions
+ * into a byte stream.
+ */
+ public enum ScalarEncoding {
+ /** Each dimension is quantized to 8 bits and treated as an unsigned value. */
+ UNSIGNED_BYTE(0, (byte) 8, 8),
+ /** Each dimension is quantized to 4 bits two values are packed into each output byte. */
+ PACKED_NIBBLE(1, (byte) 4, 4),
+ /**
+ * Each dimension is quantized to 7 bits and treated as a signed value.
+ *
+ * <p>This is intended for backwards compatibility with older iterations of scalar quantization.
+ * This setting will produce an index the same size as {@link #UNSIGNED_BYTE} but will produce
+ * less accurate vector comparisons.
+ */
+ SEVEN_BIT(2, (byte) 7, 8),
+ /**
+ * Each dimension is quantized to a single bit and packed into bytes. During query time, the
+ * query vector is quantized to 4 bits per dimension.
+ *
+ * <p>This is the most space efficient encoding, and will produce an index 8x smaller than
+ * {@link #UNSIGNED_BYTE}. However, this comes at the cost of accuracy.
+ */
+ SINGLE_BIT_QUERY_NIBBLE(3, (byte) 1, 1, (byte) 4, 4),
+ /**
+ * Each dimension is quantized to 2 bits (dibit) and packed into bytes. During query time, the
+ * query vector is quantized to 4 bits per dimension.
+ *
+ * <p>This encoding produces an index 4x smaller than {@link #UNSIGNED_BYTE}, offering a balance
+ * between the compression of {@link #SINGLE_BIT_QUERY_NIBBLE} and the accuracy of {@link
+ * #PACKED_NIBBLE}.
+ */
+ DIBIT_QUERY_NIBBLE(4, (byte) 2, 2, (byte) 4, 4) {
+ @Override
+ public int getDiscreteDimensions(int dimensions) {
+ int queryDiscretized = (dimensions * 4 + 7) / 8 * 8 / 4;
+ // we want to force dibit packing to byte boundaries assuming single bit striping
+ // so we discretize to the same as single bit encoding
+ int docDiscretized = (dimensions + 7) / 8 * 8;
+ int maxDiscretized = Math.max(queryDiscretized, docDiscretized);
+ assert maxDiscretized % (8.0 / 4) == 0
+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
+ assert maxDiscretized % (8.0 / 2) == 0
+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
+ return maxDiscretized;
+ }
+
+ @Override
+ public int getDocPackedLength(int dimensions) {
+ int discretized = getDiscreteDimensions(dimensions);
+ // DIBIT should be stored as two single bits striped
+ return 2 * ((discretized + 7) / 8);
+ }
+ };
+
+ public static ScalarEncoding fromNumBits(int bits) {
+ for (ScalarEncoding encoding : values()) {
+ if (encoding.bits == bits) {
+ return encoding;
+ }
+ }
+ throw new IllegalArgumentException("No encoding for " + bits + " bits");
+ }
+
+ /** The number used to identify this encoding on the wire, rather than relying on ordinal. */
+ private final int wireNumber;
+
+ private final byte bits, queryBits;
+ private final int bitsPerDim, queryBitsPerDim;
+
+ ScalarEncoding(int wireNumber, byte bits, int bitsPerDim) {
+ this.wireNumber = wireNumber;
+ this.bits = bits;
+ this.queryBits = bits;
+ this.bitsPerDim = bitsPerDim;
+ this.queryBitsPerDim = bitsPerDim;
+ }
+
+ ScalarEncoding(int wireNumber, byte bits, int bitsPerDim, byte queryBits, int queryBitsPerDim) {
+ this.wireNumber = wireNumber;
+ this.bits = bits;
+ this.queryBits = queryBits;
+ this.bitsPerDim = bitsPerDim;
+ this.queryBitsPerDim = queryBitsPerDim;
+ }
+
+ public boolean isAsymmetric() {
+ return bits != queryBits;
+ }
+
+ public int getWireNumber() {
+ return wireNumber;
+ }
+
+ /** Return the number of bits used per dimension. */
+ public byte getBits() {
+ return bits;
+ }
+
+ public byte getQueryBits() {
+ return queryBits;
+ }
+
+ /** Return the number of dimensions rounded up to fit into whole bytes. */
+ public int getDiscreteDimensions(int dimensions) {
+ if (queryBits == bits) {
+ int totalBits = dimensions * bitsPerDim;
+ return (totalBits + 7) / 8 * 8 / bitsPerDim;
+ }
+ int queryDiscretized = (dimensions * queryBitsPerDim + 7) / 8 * 8 / queryBitsPerDim;
+ int docDiscretized = (dimensions * bitsPerDim + 7) / 8 * 8 / bitsPerDim;
+ int maxDiscretized = Math.max(queryDiscretized, docDiscretized);
+ assert maxDiscretized % (8.0 / queryBitsPerDim) == 0
+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
+ assert maxDiscretized % (8.0 / bitsPerDim) == 0
+ : "bad discretized=" + maxDiscretized + " for dim=" + dimensions;
+ return maxDiscretized;
+ }
+
+ /** Return the number of dimensions that can be packed into a single byte. */
+ public int getDocBitsPerDim() {
+ return this.bitsPerDim;
+ }
+
+ public int getQueryBitsPerDim() {
+ return this.queryBitsPerDim;
+ }
+
+ /** Return the number of bytes required to store a packed vector of the given dimensions. */
+ public int getDocPackedLength(int dimensions) {
+ int discretized = getDiscreteDimensions(dimensions);
+ // how many bytes do we need to store the quantized vector?
+ int totalBits = discretized * bitsPerDim;
+ return (totalBits + 7) / 8;
+ }
+
+ public int getQueryPackedLength(int dimensions) {
+ int discretized = getDiscreteDimensions(dimensions);
+ // how many bytes do we need to store the quantized vector?
+ int totalBits = discretized * queryBitsPerDim;
+ return (totalBits + 7) / 8;
+ }
- public abstract float getScoreCorrectionConstant(int ord) throws IOException;
+ /** Returns the encoding for the given wire number, or empty if unknown. */
+ public static Optional<ScalarEncoding> fromWireNumber(int wireNumber) {
+ for (ScalarEncoding encoding : values()) {
+ if (encoding.wireNumber == wireNumber) {
+ return Optional.of(encoding);
+ }
+ }
+ return Optional.empty();
+ }
+ }
/**
- * Return a {@link VectorScorer} for the given query vector.
+ * Retrieve the corrective terms for the given vector ordinal. For the dot-product family of
+ * distances, the corrective terms are, in order
+ *
+ * <ul>
+ * <li>the lower optimized interval
+ * <li>the upper optimized interval
+ * <li>the dot-product of the non-centered vector with the centroid
+ * <li>the sum of quantized components
+ * </ul>
+ *
+ * For euclidean:
*
- * @param query the query vector
- * @return a {@link VectorScorer} instance or null
+ * <ul>
+ * <li>the lower optimized interval
+ * <li>the upper optimized interval
+ * <li>the l2norm of the centered vector
+ * <li>the sum of quantized components
+ * </ul>
+ *
+ * @param vectorOrd the vector ordinal
+ * @return the corrective terms
+ * @throws IOException if an I/O error occurs
*/
- public VectorScorer scorer(float[] query) throws IOException {
- throw new UnsupportedOperationException();
- }
+ public abstract OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int vectorOrd)
+ throws IOException;
- @Override
- public QuantizedByteVectorValues copy() throws IOException {
- return this;
- }
+ /**
+ * @return the quantizer used to quantize the vectors
+ */
+ public abstract OptimizedScalarQuantizer getQuantizer();
+
+ /**
+ * @return the scalar encoding used to pack the stored vectors.
+ */
+ public abstract ScalarEncoding getScalarEncoding();
+
+ /**
+ * @return the centroid used to center the vectors prior to quantization
+ */
+ public abstract float[] getCentroid() throws IOException;
+
+ /**
+ * @return the dot product of the centroid.
+ */
+ public abstract float getCentroidDP() throws IOException;
@Override
- public IndexInput getSlice() {
- return null;
- }
+ public abstract QuantizedByteVectorValues copy() throws IOException;
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java
index 7c49b1f10d8..3bc264fd91a 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedVectorsReader.java
@@ -27,7 +27,7 @@ import org.apache.lucene.util.Accountable;
*/
public interface QuantizedVectorsReader extends Closeable, Accountable {
- QuantizedByteVectorValues getQuantizedVectorValues(String fieldName) throws IOException;
+ BaseQuantizedByteVectorValues getQuantizedVectorValues(String fieldName) throws IOException;
ScalarQuantizer getQuantizationState(String fieldName);
}
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
index ade4a248c0d..c8a9efb07ac 100644
--- a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
+++ b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java
@@ -26,7 +26,7 @@ import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.BaseQuantizedByteVectorValues;
public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorer {
@@ -65,7 +65,7 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorer
private RandomVectorScorerSupplier getByteScorerSupplier(
ByteVectorValues vectorValues, VectorSimilarityFunction similarityType) throws IOException {
// a quantized values here is a wrapping or delegation issue
- assert !(vectorValues instanceof QuantizedByteVectorValues);
+ assert !(vectorValues instanceof BaseQuantizedByteVectorValues);
// currently only supports binary vectors
if (vectorValues instanceof ByteVectorValues bvv
&& bvv instanceof HasIndexSlice byteVectorValues
@@ -104,7 +104,7 @@ public class Lucene99MemorySegmentFlatVectorsScorer implements FlatVectorsScorer
throws IOException {
FlatVectorsScorer.checkDimensions(queryVector.length, vectorValues.dimension());
// a quantized values here is a wrapping or delegation issue
- assert !(vectorValues instanceof QuantizedByteVectorValues);
+ assert !(vectorValues instanceof BaseQuantizedByteVectorValues);
if (vectorValues instanceof ByteVectorValues bvv
&& bvv instanceof HasIndexSlice byteVectorValues
&& byteVectorValues.getSlice() != null) {
diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.java
index 12b95f6c2ff..de0f792ebf9 100644
--- a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentScalarQuantizedVectorScorer.java
@@ -34,7 +34,7 @@ import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
-import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.LegacyQuantizedByteVectorValues;
import org.apache.lucene.util.quantization.ScalarQuantizer;
class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsScorer {
@@ -50,7 +50,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsSco
public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)
throws IOException {
- if (vectorValues instanceof QuantizedByteVectorValues quantized
+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantized
&& quantized.getSlice() instanceof MemorySegmentAccessInput input) {
return new RandomVectorScorerSupplierImpl(similarityFunction, quantized, input);
}
@@ -61,7 +61,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsSco
public RandomVectorScorer getRandomVectorScorer(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)
throws IOException {
- if (vectorValues instanceof QuantizedByteVectorValues quantized
+ if (vectorValues instanceof LegacyQuantizedByteVectorValues quantized
&& quantized.getSlice() instanceof MemorySegmentAccessInput input) {
return new RandomVectorScorerImpl(similarityFunction, quantized, input, target);
}
@@ -94,7 +94,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsSco
RandomVectorScorerBase(
VectorSimilarityFunction similarityFunction,
- QuantizedByteVectorValues values,
+ LegacyQuantizedByteVectorValues values,
MemorySegmentAccessInput input) {
super(values);
@@ -204,7 +204,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsSco
RandomVectorScorerImpl(
VectorSimilarityFunction similarityFunction,
- QuantizedByteVectorValues values,
+ LegacyQuantizedByteVectorValues values,
MemorySegmentAccessInput input,
float[] target) {
super(similarityFunction, values, input);
@@ -250,7 +250,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsSco
private record RandomVectorScorerSupplierImpl(
VectorSimilarityFunction similarityFunction,
- QuantizedByteVectorValues values,
+ LegacyQuantizedByteVectorValues values,
MemorySegmentAccessInput input)
implements RandomVectorScorerSupplier {
@@ -272,7 +272,7 @@ class Lucene99MemorySegmentScalarQuantizedVectorScorer implements FlatVectorsSco
UpdateableRandomVectorScorerImpl(
VectorSimilarityFunction similarityFunction,
- QuantizedByteVectorValues values,
+ LegacyQuantizedByteVectorValues values,
MemorySegmentAccessInput input) {
super(similarityFunction, values, input);
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
index 3ccc5987d25..a1b64eb9203 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104HnswScalarQuantizedVectorsFormat.java
@@ -29,7 +29,6 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -51,6 +50,7 @@ import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.SameThreadExecutorService;
import org.apache.lucene.util.VectorUtil;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
import org.junit.Before;
public class TestLucene104HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
index 08a96b54bae..7825d92706e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104ScalarQuantizedVectorsFormat.java
@@ -28,7 +28,6 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
@@ -52,6 +51,8 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
import org.junit.Before;
public class TestLucene104ScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java
index 4b590d0c3ea..70729fc1946 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestFullPrecisionFloatVectorSimilarityValuesSource.java
@@ -23,7 +23,6 @@ import java.util.Objects;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -39,6 +38,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.TestVectorUtil;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
import org.junit.Before;
import org.junit.Test;
@@ -75,7 +75,7 @@ public class TestFullPrecisionFloatVectorSimilarityValuesSource extends LuceneTe
private KnnVectorsFormat getKnnFormat(int bits) {
return new Lucene104HnswScalarQuantizedVectorsFormat(
- Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.fromNumBits(bits),
+ ScalarEncoding.fromNumBits(bits),
Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
1,
diff --git a/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java b/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
index cb0a0add706..510008a4b31 100644
--- a/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
+++ b/lucene/core/src/test/org/apache/lucene/util/quantization/TestOptimizedScalarQuantizer.java
@@ -24,7 +24,6 @@ import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.deQua
import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.packAsBinary;
import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.unpackBinary;
-import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.VectorUtil;
@@ -182,7 +181,8 @@ public class TestOptimizedScalarQuantizer extends LuceneTestCase {
public void testUnpackBinary() {
int dim = randomIntBetween(1, 4096);
- ScalarEncoding encoding = ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE;
+ QuantizedByteVectorValues.ScalarEncoding encoding =
+ QuantizedByteVectorValues.ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE;
byte[] scratch = new byte[encoding.getDiscreteDimensions(dim)];
for (int i = 0; i < scratch.length; i++) {
scratch[i] = randomBoolean() ? (byte) 1 : (byte) 0;
@@ -196,7 +196,8 @@ public class TestOptimizedScalarQuantizer extends LuceneTestCase {
public void testPackTransposeDibit() {
int dim = randomIntBetween(1, 4096);
- ScalarEncoding encoding = ScalarEncoding.DIBIT_QUERY_NIBBLE;
+ QuantizedByteVectorValues.ScalarEncoding encoding =
+ QuantizedByteVectorValues.ScalarEncoding.DIBIT_QUERY_NIBBLE;
byte[] scratch = new byte[encoding.getDiscreteDimensions(dim)];
for (int i = 0; i < scratch.length; i++) {
scratch[i] = (byte) randomIntBetween(0, 3);