You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2025/10/27 14:17:42 UTC
(lucene) branch branch_10x updated: Moves the 102 legacy binary formats to backwards codecs (#15353)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 6f42ab340d2 Moves the 102 legacy binary formats to backwards codecs (#15353)
6f42ab340d2 is described below
commit 6f42ab340d26cf269763b8ccd1a84e61a64f208d
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Mon Oct 27 10:08:38 2025 -0400
Moves the 102 legacy binary formats to backwards codecs (#15353)
Users should utilize the new 104 scalar quantization format with the asymmetric quantization encoding.
When migrating to the new format, the main change will be instead of using `new Lucene102HnswBinaryQuantizedVectorsFormat()`, user's should use `new Lucene104HnswScalarQuantizedVectorsFormat(ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE, DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null, HNSW_GRAPH_THRESHOLD)`
---
lucene/CHANGES.txt | 6 +-
lucene/backward-codecs/src/java/module-info.java | 5 +-
.../lucene102/BinarizedByteVectorValues.java | 10 +-
.../Lucene102BinaryFlatVectorsScorer.java | 86 +---
.../Lucene102BinaryQuantizedVectorsFormat.java | 16 +-
.../Lucene102BinaryQuantizedVectorsReader.java | 20 +-
.../Lucene102HnswBinaryQuantizedVectorsFormat.java | 37 +-
.../lucene102/OffHeapBinarizedVectorValues.java | 2 +-
.../backward_codecs/lucene102/package-info.java | 19 +
.../org.apache.lucene.codecs.KnnVectorsFormat | 2 +
.../Lucene102BinaryQuantizedVectorsWriter.java | 16 +-
.../Lucene102RWBinaryFlatVectorsScorer.java} | 60 +--
.../Lucene102RWBinaryQuantizedVectorsFormat.java | 34 ++
...ucene102RWHnswBinaryQuantizedVectorsFormat.java | 59 +++
.../TestLucene102BinaryQuantizedVectorsFormat.java | 6 +-
...tLucene102HnswBinaryQuantizedVectorsFormat.java | 22 +-
lucene/core/src/java/module-info.java | 3 -
.../lucene/codecs/lucene102/package-info.java | 437 ---------------------
.../Lucene104HnswScalarQuantizedVectorsFormat.java | 12 +
.../org.apache.lucene.codecs.KnnVectorsFormat | 2 -
20 files changed, 221 insertions(+), 633 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index aaeff2558c6..b95ec1e5954 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -35,9 +35,11 @@ New Features
`Lucene104HnswScalarQuantizedVectorsFormat` replaces the now legacy `Lucene99HnswScalarQuantizedVectorsFormat`
(Trevor McCulloch)
- * GITHUB#15271: Extend `Lucene104ScalarQuantizedVectorsFormat` and `Lucene104HnswScalarQuantizedVectorsFormat` to
+ * GITHUB#15271, GITHUB#15353: Extend `Lucene104ScalarQuantizedVectorsFormat` and `Lucene104HnswScalarQuantizedVectorsFormat` to
allow asymmetric quantization. The initially supported bits are single bit with 4 bit queries. This is a replacement
- for the now legacy `Lucene102HnswBinaryQuantizedVectorsFormat` and `Lucene102BinaryQuantizedVectorsFormat`.
+ for the now legacy `Lucene102HnswBinaryQuantizedVectorsFormat` and `Lucene102BinaryQuantizedVectorsFormat`. To use
+ the new format for asymmetric binary quantization,
+ `Lucene104HnswScalarQuantizedVectorsFormat(ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE, int, int)`
(Ben Trent)
Improvements
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
index 39d028232c5..dac0bb2f59f 100644
--- a/lucene/backward-codecs/src/java/module-info.java
+++ b/lucene/backward-codecs/src/java/module-info.java
@@ -40,6 +40,7 @@ module org.apache.lucene.backward_codecs {
exports org.apache.lucene.backward_codecs.lucene912;
exports org.apache.lucene.backward_codecs.lucene100;
exports org.apache.lucene.backward_codecs.lucene101;
+ exports org.apache.lucene.backward_codecs.lucene102;
exports org.apache.lucene.backward_codecs.lucene103;
exports org.apache.lucene.backward_codecs.packed;
exports org.apache.lucene.backward_codecs.store;
@@ -61,7 +62,9 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat,
- org.apache.lucene.backward_codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
+ org.apache.lucene.backward_codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat,
+ org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat,
+ org.apache.lucene.backward_codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat;
provides org.apache.lucene.codecs.Codec with
org.apache.lucene.backward_codecs.lucene80.Lucene80Codec,
org.apache.lucene.backward_codecs.lucene84.Lucene84Codec,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/BinarizedByteVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/BinarizedByteVectorValues.java
similarity index 88%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene102/BinarizedByteVectorValues.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/BinarizedByteVectorValues.java
index 62eff4d72ac..d19ba69866c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/BinarizedByteVectorValues.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/BinarizedByteVectorValues.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.discretize;
@@ -51,15 +51,15 @@ abstract class BinarizedByteVectorValues extends ByteVectorValues {
* @return the corrective terms
* @throws IOException if an I/O error occurs
*/
- public abstract OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int vectorOrd)
+ abstract OptimizedScalarQuantizer.QuantizationResult getCorrectiveTerms(int vectorOrd)
throws IOException;
/**
* @return the quantizer used to quantize the vectors
*/
- public abstract OptimizedScalarQuantizer getQuantizer();
+ abstract OptimizedScalarQuantizer getQuantizer();
- public abstract float[] getCentroid() throws IOException;
+ abstract float[] getCentroid() throws IOException;
int discretizedDimensions() {
return discretize(dimension(), 64);
@@ -71,7 +71,7 @@ abstract class BinarizedByteVectorValues extends ByteVectorValues {
* @param query the query vector
* @return a {@link VectorScorer} instance or null
*/
- public abstract VectorScorer scorer(float[] query) throws IOException;
+ abstract VectorScorer scorer(float[] query) throws IOException;
@Override
public abstract BinarizedByteVectorValues copy() throws IOException;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java
similarity index 63%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java
index 02e37a9e89d..9732ca36598 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java
@@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
-import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.QUERY_BITS;
+import static org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.QUERY_BITS;
import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;
import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT;
@@ -30,15 +30,22 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.VectorUtil;
import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer.QuantizationResult;
/** Vector scorer over binarized vector values */
public class Lucene102BinaryFlatVectorsScorer implements FlatVectorsScorer {
- private final FlatVectorsScorer nonQuantizedDelegate;
- private static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1);
+ /** The delegate scorer for non-quantized vectors */
+ protected final FlatVectorsScorer nonQuantizedDelegate;
+ /** Scaling factor for 4-bit quantization */
+ protected static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1);
+
+ /**
+ * Construct a new scorer
+ *
+ * @param nonQuantizedDelegate the delegate scorer for non-quantized vectors
+ */
public Lucene102BinaryFlatVectorsScorer(FlatVectorsScorer nonQuantizedDelegate) {
this.nonQuantizedDelegate = nonQuantizedDelegate;
}
@@ -47,11 +54,7 @@ public class Lucene102BinaryFlatVectorsScorer implements FlatVectorsScorer {
public RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues)
throws IOException {
- if (vectorValues instanceof BinarizedByteVectorValues) {
- throw new UnsupportedOperationException(
- "getRandomVectorScorerSupplier(VectorSimilarityFunction,RandomAccessVectorValues) not implemented for binarized format");
- }
- return nonQuantizedDelegate.getRandomVectorScorerSupplier(similarityFunction, vectorValues);
+ throw new UnsupportedOperationException("Old codecs may only be used for reading");
}
@Override
@@ -69,7 +72,7 @@ public class Lucene102BinaryFlatVectorsScorer implements FlatVectorsScorer {
target = copy;
byte[] initial = new byte[target.length];
byte[] quantized = new byte[QUERY_BITS * binarizedVectors.discretizedDimensions() / 8];
- OptimizedScalarQuantizer.QuantizationResult queryCorrections =
+ QuantizationResult queryCorrections =
quantizer.scalarQuantize(target, initial, (byte) 4, centroid);
transposeHalfByte(initial, quantized);
return new RandomVectorScorer.AbstractRandomVectorScorer(binarizedVectors) {
@@ -90,78 +93,21 @@ public class Lucene102BinaryFlatVectorsScorer implements FlatVectorsScorer {
return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
}
- RandomVectorScorerSupplier getRandomVectorScorerSupplier(
- VectorSimilarityFunction similarityFunction,
- Lucene102BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues scoringVectors,
- BinarizedByteVectorValues targetVectors) {
- return new BinarizedRandomVectorScorerSupplier(
- scoringVectors, targetVectors, similarityFunction);
- }
-
@Override
public String toString() {
return "Lucene102BinaryFlatVectorsScorer(nonQuantizedDelegate=" + nonQuantizedDelegate + ")";
}
- /** Vector scorer supplier over binarized vector values */
- static class BinarizedRandomVectorScorerSupplier implements RandomVectorScorerSupplier {
- private final Lucene102BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues
- queryVectors;
- private final BinarizedByteVectorValues targetVectors;
- private final VectorSimilarityFunction similarityFunction;
-
- BinarizedRandomVectorScorerSupplier(
- Lucene102BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors,
- BinarizedByteVectorValues targetVectors,
- VectorSimilarityFunction similarityFunction) {
- this.queryVectors = queryVectors;
- this.targetVectors = targetVectors;
- this.similarityFunction = similarityFunction;
- }
-
- @Override
- public UpdateableRandomVectorScorer scorer() throws IOException {
- final BinarizedByteVectorValues targetVectors = this.targetVectors.copy();
- final Lucene102BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors =
- this.queryVectors.copy();
- return new UpdateableRandomVectorScorer.AbstractUpdateableRandomVectorScorer(targetVectors) {
- private QuantizationResult queryCorrections = null;
- private byte[] vector = null;
-
- @Override
- public void setScoringOrdinal(int node) throws IOException {
- queryCorrections = queryVectors.getCorrectiveTerms(node);
- vector = queryVectors.vectorValue(node);
- }
-
- @Override
- public float score(int node) throws IOException {
- if (vector == null || queryCorrections == null) {
- throw new IllegalStateException("setScoringOrdinal was not called");
- }
- return quantizedScore(vector, queryCorrections, targetVectors, node, similarityFunction);
- }
- };
- }
-
- @Override
- public RandomVectorScorerSupplier copy() throws IOException {
- return new BinarizedRandomVectorScorerSupplier(
- queryVectors.copy(), targetVectors.copy(), similarityFunction);
- }
- }
-
static float quantizedScore(
byte[] quantizedQuery,
- OptimizedScalarQuantizer.QuantizationResult queryCorrections,
+ QuantizationResult queryCorrections,
BinarizedByteVectorValues targetVectors,
int targetOrd,
VectorSimilarityFunction similarityFunction)
throws IOException {
byte[] binaryCode = targetVectors.vectorValue(targetOrd);
float qcDist = VectorUtil.int4BitDotProduct(quantizedQuery, binaryCode);
- OptimizedScalarQuantizer.QuantizationResult indexCorrections =
- targetVectors.getCorrectiveTerms(targetOrd);
+ QuantizationResult indexCorrections = targetVectors.getCorrectiveTerms(targetOrd);
float x1 = indexCorrections.quantizedComponentSum();
float ax = indexCorrections.lowerInterval();
// Here we assume `lx` is simply bit vectors, so the scaling isn't necessary
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java
similarity index 92%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java
index 1f08dd68bf9..16251990634 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
import java.io.IOException;
import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
@@ -89,11 +89,11 @@ import org.apache.lucene.index.SegmentWriteState;
*/
public class Lucene102BinaryQuantizedVectorsFormat extends FlatVectorsFormat {
- public static final byte QUERY_BITS = 4;
- public static final byte INDEX_BITS = 1;
+ static final byte QUERY_BITS = 4;
+ static final byte INDEX_BITS = 1;
- public static final String BINARIZED_VECTOR_COMPONENT = "BVEC";
- public static final String NAME = "Lucene102BinaryQuantizedVectorsFormat";
+ static final String BINARIZED_VECTOR_COMPONENT = "BVEC";
+ static final String NAME = "Lucene102BinaryQuantizedVectorsFormat";
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
@@ -103,7 +103,8 @@ public class Lucene102BinaryQuantizedVectorsFormat extends FlatVectorsFormat {
static final String VECTOR_DATA_EXTENSION = "veb";
static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16;
- private static final FlatVectorsFormat rawVectorFormat =
+ /** The raw (unquantized) vector format used to read the original vectors. */
+ protected static final FlatVectorsFormat rawVectorFormat =
new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());
private static final Lucene102BinaryFlatVectorsScorer scorer =
@@ -116,8 +117,7 @@ public class Lucene102BinaryQuantizedVectorsFormat extends FlatVectorsFormat {
@Override
public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new Lucene102BinaryQuantizedVectorsWriter(
- scorer, rawVectorFormat.fieldsWriter(state), state);
+ throw new UnsupportedOperationException("Old codecs may only be used for reading");
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
similarity index 95%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
index 349df3049c0..4d77add4609 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
@@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
-import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION;
+import static org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding;
import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.discretize;
@@ -55,7 +55,7 @@ import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
/** Reader for binary quantized vectors in the Lucene 10.2 format. */
-class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader {
+public class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader {
private static final long SHALLOW_SIZE =
RamUsageEstimator.shallowSizeOfInstance(Lucene102BinaryQuantizedVectorsReader.class);
@@ -65,7 +65,15 @@ class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader {
private final FlatVectorsReader rawVectorsReader;
private final Lucene102BinaryFlatVectorsScorer vectorScorer;
- Lucene102BinaryQuantizedVectorsReader(
+ /**
+ * Creates a new reader for binary quantized vectors.
+ *
+ * @param state the segment read state
+ * @param rawVectorsReader the reader for the raw (non-quantized) vectors
+ * @param vectorsScorer the scorer for binary quantized vectors
+ * @throws IOException if an I/O error occurs
+ */
+ public Lucene102BinaryQuantizedVectorsReader(
SegmentReadState state,
FlatVectorsReader rawVectorsReader,
Lucene102BinaryFlatVectorsScorer vectorsScorer)
@@ -101,7 +109,7 @@ class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader {
openDataInput(
state,
versionMeta,
- Lucene102BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION,
+ VECTOR_DATA_EXTENSION,
Lucene102BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,
// Quantized vectors are accessed randomly from their node ID stored in the HNSW
// graph.
@@ -277,7 +285,7 @@ class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader {
return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant);
}
- public float[] getCentroid(String field) {
+ float[] getCentroid(String field) {
FieldEntry fieldEntry = fields.get(field);
if (fieldEntry != null) {
return fieldEntry.centroid;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java
similarity index 84%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java
index 465310a992f..698cca05b05 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
@@ -30,7 +30,6 @@ import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.TaskExecutor;
@@ -42,43 +41,36 @@ import org.apache.lucene.util.hnsw.HnswGraph;
*/
public class Lucene102HnswBinaryQuantizedVectorsFormat extends KnnVectorsFormat {
- public static final String NAME = "Lucene102HnswBinaryQuantizedVectorsFormat";
+ static final String NAME = "Lucene102HnswBinaryQuantizedVectorsFormat";
/**
* Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to
* {@link Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details.
*/
- private final int maxConn;
+ protected final int maxConn;
/**
* The number of candidate neighbors to track while searching the graph for each newly inserted
* node. Defaults to {@link Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link HnswGraph}
* for details.
*/
- private final int beamWidth;
+ protected final int beamWidth;
/** The format for storing, reading, merging vectors on disk */
- private static final FlatVectorsFormat flatVectorsFormat =
+ protected static final FlatVectorsFormat flatVectorsFormat =
new Lucene102BinaryQuantizedVectorsFormat();
- private final int numMergeWorkers;
- private final TaskExecutor mergeExec;
+ /** Number of workers (threads) that will be used when doing merge. */
+ protected final int numMergeWorkers;
+
+ /** The {@link TaskExecutor} that will be used to do merge. */
+ protected final TaskExecutor mergeExec;
/** Constructs a format using default graph construction parameters */
public Lucene102HnswBinaryQuantizedVectorsFormat() {
this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null);
}
- /**
- * Constructs a format using the given graph construction parameters.
- *
- * @param maxConn the maximum number of connections to a node in the HNSW graph
- * @param beamWidth the size of the queue maintained during graph construction.
- */
- public Lucene102HnswBinaryQuantizedVectorsFormat(int maxConn, int beamWidth) {
- this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null);
- }
-
/**
* Constructs a format using the given graph construction parameters and scalar quantization.
*
@@ -122,14 +114,7 @@ public class Lucene102HnswBinaryQuantizedVectorsFormat extends KnnVectorsFormat
@Override
public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new Lucene99HnswVectorsWriter(
- state,
- maxConn,
- beamWidth,
- flatVectorsFormat.fieldsWriter(state),
- numMergeWorkers,
- mergeExec,
- 0);
+ throw new UnsupportedOperationException("Old codecs may only be used for reading");
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/OffHeapBinarizedVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/OffHeapBinarizedVectorValues.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene102/OffHeapBinarizedVectorValues.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/OffHeapBinarizedVectorValues.java
index 13eef8b263b..bfd1046693a 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/OffHeapBinarizedVectorValues.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/OffHeapBinarizedVectorValues.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.discretize;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/package-info.java
new file mode 100644
index 00000000000..3c45174af89
--- /dev/null
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Lucene 10.2 file format. */
+package org.apache.lucene.backward_codecs.lucene102;
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
index f8b18e7e539..8f0825e3f1b 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
@@ -20,3 +20,5 @@ org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat
org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat
org.apache.lucene.backward_codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat
org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat
+org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat
+org.apache.lucene.backward_codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
index fc42e93bf9a..70a89b3a910 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsWriter.java
@@ -14,12 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
-import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.BINARIZED_VECTOR_COMPONENT;
-import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
-import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.INDEX_BITS;
-import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.QUERY_BITS;
+import static org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.BINARIZED_VECTOR_COMPONENT;
+import static org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
+import static org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.INDEX_BITS;
+import static org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.QUERY_BITS;
import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance;
@@ -63,7 +63,7 @@ import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
/** Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 */
-public class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {
+class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {
private static final long SHALLOW_RAM_BYTES_USED =
shallowSizeOfInstance(Lucene102BinaryQuantizedVectorsWriter.class);
@@ -71,7 +71,7 @@ public class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {
private final List<FieldWriter> fields = new ArrayList<>();
private final IndexOutput meta, binarizedVectorData;
private final FlatVectorsWriter rawVectorDelegate;
- private final Lucene102BinaryFlatVectorsScorer vectorsScorer;
+ private final Lucene102RWBinaryFlatVectorsScorer vectorsScorer;
private boolean finished;
/**
@@ -80,7 +80,7 @@ public class Lucene102BinaryQuantizedVectorsWriter extends FlatVectorsWriter {
* @param vectorsScorer the scorer to use for scoring vectors
*/
protected Lucene102BinaryQuantizedVectorsWriter(
- Lucene102BinaryFlatVectorsScorer vectorsScorer,
+ Lucene102RWBinaryFlatVectorsScorer vectorsScorer,
FlatVectorsWriter rawVectorDelegate,
SegmentWriteState state)
throws IOException {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWBinaryFlatVectorsScorer.java
similarity index 70%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWBinaryFlatVectorsScorer.java
index 02e37a9e89d..6e1b6f69804 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/Lucene102BinaryFlatVectorsScorer.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWBinaryFlatVectorsScorer.java
@@ -14,33 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
-import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.QUERY_BITS;
-import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN;
import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT;
-import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.transposeHalfByte;
import java.io.IOException;
import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.VectorUtil;
-import org.apache.lucene.util.hnsw.RandomVectorScorer;
import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
-import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer.QuantizationResult;
/** Vector scorer over binarized vector values */
-public class Lucene102BinaryFlatVectorsScorer implements FlatVectorsScorer {
- private final FlatVectorsScorer nonQuantizedDelegate;
- private static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1);
+public class Lucene102RWBinaryFlatVectorsScorer extends Lucene102BinaryFlatVectorsScorer {
- public Lucene102BinaryFlatVectorsScorer(FlatVectorsScorer nonQuantizedDelegate) {
- this.nonQuantizedDelegate = nonQuantizedDelegate;
+ /**
+ * @param nonQuantizedDelegate the delegate scorer for non-quantized vectors
+ */
+ public Lucene102RWBinaryFlatVectorsScorer(FlatVectorsScorer nonQuantizedDelegate) {
+ super(nonQuantizedDelegate);
}
@Override
@@ -54,42 +49,6 @@ public class Lucene102BinaryFlatVectorsScorer implements FlatVectorsScorer {
return nonQuantizedDelegate.getRandomVectorScorerSupplier(similarityFunction, vectorValues);
}
- @Override
- public RandomVectorScorer getRandomVectorScorer(
- VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, float[] target)
- throws IOException {
- if (vectorValues instanceof BinarizedByteVectorValues binarizedVectors) {
- OptimizedScalarQuantizer quantizer = binarizedVectors.getQuantizer();
- float[] centroid = binarizedVectors.getCentroid();
- // We make a copy as the quantization process mutates the input
- float[] copy = ArrayUtil.copyOfSubArray(target, 0, target.length);
- if (similarityFunction == COSINE) {
- VectorUtil.l2normalize(copy);
- }
- target = copy;
- byte[] initial = new byte[target.length];
- byte[] quantized = new byte[QUERY_BITS * binarizedVectors.discretizedDimensions() / 8];
- OptimizedScalarQuantizer.QuantizationResult queryCorrections =
- quantizer.scalarQuantize(target, initial, (byte) 4, centroid);
- transposeHalfByte(initial, quantized);
- return new RandomVectorScorer.AbstractRandomVectorScorer(binarizedVectors) {
- @Override
- public float score(int node) throws IOException {
- return quantizedScore(
- quantized, queryCorrections, binarizedVectors, node, similarityFunction);
- }
- };
- }
- return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
- }
-
- @Override
- public RandomVectorScorer getRandomVectorScorer(
- VectorSimilarityFunction similarityFunction, KnnVectorValues vectorValues, byte[] target)
- throws IOException {
- return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target);
- }
-
RandomVectorScorerSupplier getRandomVectorScorerSupplier(
VectorSimilarityFunction similarityFunction,
Lucene102BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues scoringVectors,
@@ -153,15 +112,14 @@ public class Lucene102BinaryFlatVectorsScorer implements FlatVectorsScorer {
static float quantizedScore(
byte[] quantizedQuery,
- OptimizedScalarQuantizer.QuantizationResult queryCorrections,
+ QuantizationResult queryCorrections,
BinarizedByteVectorValues targetVectors,
int targetOrd,
VectorSimilarityFunction similarityFunction)
throws IOException {
byte[] binaryCode = targetVectors.vectorValue(targetOrd);
float qcDist = VectorUtil.int4BitDotProduct(quantizedQuery, binaryCode);
- OptimizedScalarQuantizer.QuantizationResult indexCorrections =
- targetVectors.getCorrectiveTerms(targetOrd);
+ QuantizationResult indexCorrections = targetVectors.getCorrectiveTerms(targetOrd);
float x1 = indexCorrections.quantizedComponentSum();
float ax = indexCorrections.lowerInterval();
// Here we assume `lx` is simply bit vectors, so the scaling isn't necessary
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWBinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWBinaryQuantizedVectorsFormat.java
new file mode 100644
index 00000000000..4bb5a2e1c2f
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWBinaryQuantizedVectorsFormat.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene102;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
+import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
+import org.apache.lucene.index.SegmentWriteState;
+
+public class Lucene102RWBinaryQuantizedVectorsFormat extends Lucene102BinaryQuantizedVectorsFormat {
+
+ private static final Lucene102RWBinaryFlatVectorsScorer rwScorer =
+ new Lucene102RWBinaryFlatVectorsScorer(FlatVectorScorerUtil.getLucene99FlatVectorsScorer());
+
+ @Override
+ public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ return new Lucene102BinaryQuantizedVectorsWriter(
+ rwScorer, rawVectorFormat.fieldsWriter(state), state);
+ }
+}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWHnswBinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWHnswBinaryQuantizedVectorsFormat.java
new file mode 100644
index 00000000000..9b175b86ffe
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/Lucene102RWHnswBinaryQuantizedVectorsFormat.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene102;
+
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
+import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;
+import org.apache.lucene.index.SegmentWriteState;
+
+/**
+ * A vectors format that uses HNSW graph to store and search for vectors. But vectors are binary
+ * quantized using {@link Lucene102BinaryQuantizedVectorsFormat} before being stored in the graph.
+ */
+public class Lucene102RWHnswBinaryQuantizedVectorsFormat
+ extends Lucene102HnswBinaryQuantizedVectorsFormat {
+
+ private static final FlatVectorsFormat writeFormat =
+ new Lucene102RWBinaryQuantizedVectorsFormat();
+
+ /** Constructs a format using default graph construction parameters */
+ public Lucene102RWHnswBinaryQuantizedVectorsFormat() {
+ this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null);
+ }
+
+ public Lucene102RWHnswBinaryQuantizedVectorsFormat(int maxConn, int beamWidth) {
+ this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null);
+ }
+
+ public Lucene102RWHnswBinaryQuantizedVectorsFormat(
+ int maxConn, int beamWidth, int numMergeWorkers, ExecutorService mergeExec) {
+ super(maxConn, beamWidth, numMergeWorkers, mergeExec);
+ }
+
+ @Override
+ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+ return new Lucene99HnswVectorsWriter(
+ state, maxConn, beamWidth, writeFormat.fieldsWriter(state), numMergeWorkers, mergeExec, 0);
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene102/TestLucene102BinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102BinaryQuantizedVectorsFormat.java
similarity index 96%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene102/TestLucene102BinaryQuantizedVectorsFormat.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102BinaryQuantizedVectorsFormat.java
index 1b5f206ddfe..f2b07786967 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene102/TestLucene102BinaryQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102BinaryQuantizedVectorsFormat.java
@@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
import static java.lang.String.format;
-import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.INDEX_BITS;
+import static org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.INDEX_BITS;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.discretize;
import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.packAsBinary;
@@ -51,7 +51,7 @@ import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
public class TestLucene102BinaryQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {
- private static final KnnVectorsFormat FORMAT = new Lucene102BinaryQuantizedVectorsFormat();
+ private static final KnnVectorsFormat FORMAT = new Lucene102RWBinaryQuantizedVectorsFormat();
@Override
protected Codec getCodec() {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java
similarity index 90%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java
index 31799f09669..36e336a686a 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.backward_codecs.lucene102;
import static java.lang.String.format;
import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
@@ -51,7 +51,7 @@ import org.apache.lucene.util.SameThreadExecutorService;
public class TestLucene102HnswBinaryQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {
- private static final KnnVectorsFormat FORMAT = new Lucene102HnswBinaryQuantizedVectorsFormat();
+ private static final KnnVectorsFormat FORMAT = new Lucene102RWHnswBinaryQuantizedVectorsFormat();
@Override
protected Codec getCodec() {
@@ -63,7 +63,7 @@ public class TestLucene102HnswBinaryQuantizedVectorsFormat extends BaseKnnVector
new FilterCodec("foo", Codec.getDefault()) {
@Override
public KnnVectorsFormat knnVectorsFormat() {
- return new Lucene102HnswBinaryQuantizedVectorsFormat(10, 20, 1, null);
+ return new Lucene102RWHnswBinaryQuantizedVectorsFormat(10, 20, 1, null);
}
};
String expectedPattern =
@@ -122,24 +122,26 @@ public class TestLucene102HnswBinaryQuantizedVectorsFormat extends BaseKnnVector
public void testLimits() {
expectThrows(
IllegalArgumentException.class,
- () -> new Lucene102HnswBinaryQuantizedVectorsFormat(-1, 20));
+ () -> new Lucene102RWHnswBinaryQuantizedVectorsFormat(-1, 20));
expectThrows(
- IllegalArgumentException.class, () -> new Lucene102HnswBinaryQuantizedVectorsFormat(0, 20));
+ IllegalArgumentException.class,
+ () -> new Lucene102RWHnswBinaryQuantizedVectorsFormat(0, 20));
expectThrows(
- IllegalArgumentException.class, () -> new Lucene102HnswBinaryQuantizedVectorsFormat(20, 0));
+ IllegalArgumentException.class,
+ () -> new Lucene102RWHnswBinaryQuantizedVectorsFormat(20, 0));
expectThrows(
IllegalArgumentException.class,
- () -> new Lucene102HnswBinaryQuantizedVectorsFormat(20, -1));
+ () -> new Lucene102RWHnswBinaryQuantizedVectorsFormat(20, -1));
expectThrows(
IllegalArgumentException.class,
- () -> new Lucene102HnswBinaryQuantizedVectorsFormat(512 + 1, 20));
+ () -> new Lucene102RWHnswBinaryQuantizedVectorsFormat(512 + 1, 20));
expectThrows(
IllegalArgumentException.class,
- () -> new Lucene102HnswBinaryQuantizedVectorsFormat(20, 3201));
+ () -> new Lucene102RWHnswBinaryQuantizedVectorsFormat(20, 3201));
expectThrows(
IllegalArgumentException.class,
() ->
- new Lucene102HnswBinaryQuantizedVectorsFormat(
+ new Lucene102RWHnswBinaryQuantizedVectorsFormat(
20, 100, 1, new SameThreadExecutorService()));
}
diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java
index 108f8bcadc6..9e2acc3caac 100644
--- a/lucene/core/src/java/module-info.java
+++ b/lucene/core/src/java/module-info.java
@@ -30,7 +30,6 @@ module org.apache.lucene.core {
exports org.apache.lucene.codecs.lucene94;
exports org.apache.lucene.codecs.lucene95;
exports org.apache.lucene.codecs.lucene99;
- exports org.apache.lucene.codecs.lucene102;
exports org.apache.lucene.codecs.lucene103.blocktree;
exports org.apache.lucene.codecs.lucene104;
exports org.apache.lucene.codecs.perfield;
@@ -84,8 +83,6 @@ module org.apache.lucene.core {
org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat,
- org.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat,
- org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat,
org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat,
org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
provides org.apache.lucene.codecs.PostingsFormat with
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
deleted file mode 100644
index 335a8d24813..00000000000
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
+++ /dev/null
@@ -1,437 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Lucene 10.2 file format.
- *
- * <h2>Apache Lucene - Index File Formats</h2>
- *
- * <div>
- *
- * <ul>
- * <li><a href="#Introduction">Introduction</a>
- * <li><a href="#Definitions">Definitions</a>
- * <ul>
- * <li><a href="#Inverted_Indexing">Inverted Indexing</a>
- * <li><a href="#Types_of_Fields">Types of Fields</a>
- * <li><a href="#Segments">Segments</a>
- * <li><a href="#Document_Numbers">Document Numbers</a>
- * </ul>
- * <li><a href="#Overview">Index Structure Overview</a>
- * <li><a href="#File_Naming">File Naming</a>
- * <li><a href="#file-names">Summary of File Extensions</a>
- * <ul>
- * <li><a href="#Lock_File">Lock File</a>
- * <li><a href="#History">History</a>
- * <li><a href="#Limitations">Limitations</a>
- * </ul>
- * </ul>
- *
- * </div> <a id="Introduction"></a>
- *
- * <h3>Introduction</h3>
- *
- * <div>
- *
- * <p>This document defines the index file formats used in this version of Lucene. If you are using
- * a different version of Lucene, please consult the copy of <code>docs/</code> that was distributed
- * with the version you are using.
- *
- * <p>This document attempts to provide a high-level definition of the Apache Lucene file formats.
- * </div> <a id="Definitions"></a>
- *
- * <h3>Definitions</h3>
- *
- * <div>
- *
- * <p>The fundamental concepts in Lucene are index, document, field and term.
- *
- * <p>An index contains a sequence of documents.
- *
- * <ul>
- * <li>A document is a sequence of fields.
- * <li>A field is a named sequence of terms.
- * <li>A term is a sequence of bytes.
- * </ul>
- *
- * <p>The same sequence of bytes in two different fields is considered a different term. Thus terms
- * are represented as a pair: the string naming the field, and the bytes within the field. <a
- * id="Inverted_Indexing"></a>
- *
- * <h4>Inverted Indexing</h4>
- *
- * <p>Lucene's index stores terms and statistics about those terms in order to make term-based
- * search more efficient. Lucene's terms index falls into the family of indexes known as an
- * <i>inverted index.</i> This is because it can list, for a term, the documents that contain it.
- * This is the inverse of the natural relationship, in which documents list terms. <a
- * id="Types_of_Fields"></a>
- *
- * <h4>Types of Fields</h4>
- *
- * <p>In Lucene, fields may be <i>stored</i>, in which case their text is stored in the index
- * literally, in a non-inverted manner. Fields that are inverted are called <i>indexed</i>. A field
- * may be both stored and indexed.
- *
- * <p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the text of a field
- * may be used literally as a term to be indexed. Most fields are tokenized, but sometimes it is
- * useful for certain identifier fields to be indexed literally.
- *
- * <p>See the {@link org.apache.lucene.document.Field Field} java docs for more information on
- * Fields. <a id="Segments"></a>
- *
- * <h4>Segments</h4>
- *
- * <p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>. Each segment is a
- * fully independent index, which could be searched separately. Indexes evolve by:
- *
- * <ol>
- * <li>Creating new segments for newly added documents.
- * <li>Merging existing segments.
- * </ol>
- *
- * <p>Searches may involve multiple segments and/or multiple indexes, each index potentially
- * composed of a set of segments. <a id="Document_Numbers"></a>
- *
- * <h4>Document Numbers</h4>
- *
- * <p>Internally, Lucene refers to documents by an integer <i>document number</i>. The first
- * document added to an index is numbered zero, and each subsequent document added gets a number one
- * greater than the previous.
- *
- * <p>Note that a document's number may change, so caution should be taken when storing these
- * numbers outside of Lucene. In particular, numbers may change in the following situations:
- *
- * <ul>
- * <li>
- * <p>The numbers stored in each segment are unique only within the segment, and must be
- * converted before they can be used in a larger context. The standard technique is to
- * allocate each segment a range of values, based on the range of numbers used in that
- * segment. To convert a document number from a segment to an external value, the segment's
- * <i>base</i> document number is added. To convert an external value back to a
- * segment-specific value, the segment is identified by the range that the external value is
- * in, and the segment's base value is subtracted. For example two five document segments
- * might be combined, so that the first segment has a base value of zero, and the second of
- * five. Document three from the second segment would have an external value of eight.
- * <li>
- * <p>When documents are deleted, gaps are created in the numbering. These are eventually
- * removed as the index evolves through merging. Deleted documents are dropped when segments
- * are merged. A freshly-merged segment thus has no gaps in its numbering.
- * </ul>
- *
- * </div> <a id="Overview"></a>
- *
- * <h3>Index Structure Overview</h3>
- *
- * <div>
- *
- * <p>Each segment index maintains the following:
- *
- * <ul>
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat Segment info}. This
- * contains metadata about a segment, such as the number of documents, what files it uses, and
- * information about how the segment is sorted
- * <li>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Field names}. This
- * contains metadata about the set of named fields used in the index.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Stored Field values}.
- * This contains, for each document, a list of attribute-value pairs, where the attributes are
- * field names. These are used to store auxiliary information about the document, such as its
- * title, url, or an identifier to access a database. The set of stored fields are what is
- * returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term dictionary}. A
- * dictionary containing all of the terms used in all of the indexed fields of all of the
- * documents. The dictionary also contains the number of documents which contain the term, and
- * pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Frequency data}. For
- * each term in the dictionary, the numbers of all the documents that contain that term, and
- * the frequency of the term in that document, unless frequencies are omitted ({@link
- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Proximity data}. For
- * each term in the dictionary, the positions that the term occurs in each document. Note that
- * this will not exist if all fields in all documents omit position data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
- * each field in each document, a value is stored that is multiplied into the score for hits
- * on that field.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vectors}. For each
- * field in each document, the term vector (sometimes called document vector) may be stored. A
- * term vector consists of term text and term frequency. To add Term Vectors to your index see
- * the {@link org.apache.lucene.document.Field Field} constructors
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-document values}. Like
- * stored values, these are also keyed by document number, but are generally intended to be
- * loaded into main memory for fast access. Whereas stored values are generally intended for
- * summary results from searches, per-document values are useful for things like scoring
- * factors.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live documents}. An
- * optional file indicating which documents are live.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}. Optional pair
- * of files, recording dimensionally indexed fields, to enable fast numeric range filtering
- * and large numeric values like BigInteger and BigDecimal (1D) and geographic shape
- * intersection (2D, 3D).
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}. The
- * vector format stores numeric vectors in a format optimized for random access and
- * computation, supporting high-dimensional nearest-neighbor search.
- * </ul>
- *
- * <p>Details on each of these are provided in their linked pages. </div> <a id="File_Naming"></a>
- *
- * <h3>File Naming</h3>
- *
- * <div>
- *
- * <p>All files belonging to a segment have the same name with varying extensions. The extensions
- * correspond to the different file formats described below. When using the Compound File format
- * (default for small segments) these files (except for the Segment info file, the Lock file, and
- * Deleted documents file) are collapsed into a single .cfs file (see below for details)
- *
- * <p>Typically, all segments in an index are stored in a single directory, although this is not
- * required.
- *
- * <p>File names are never re-used. That is, when any file is saved to the Directory it is given a
- * never before used filename. This is achieved using a simple generations approach. For example,
- * the first segments file is segments_1, then segments_2, etc. The generation is a sequential long
- * integer represented in alpha-numeric (base 36) form. </div> <a id="file-names"></a>
- *
- * <h3>Summary of File Extensions</h3>
- *
- * <div>
- *
- * <p>The following table summarizes the names and extensions of the files in Lucene:
- *
- * <table class="padding4" style="border-spacing: 1px; border-collapse: separate">
- * <caption>lucene filenames by extension</caption>
- * <tr>
- * <th>Name</th>
- * <th>Extension</th>
- * <th>Brief Description</th>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>
- * <td>segments_N</td>
- * <td>Stores information about a commit point</td>
- * </tr>
- * <tr>
- * <td><a href="#Lock_File">Lock File</a></td>
- * <td>write.lock</td>
- * <td>The Write lock prevents multiple IndexWriters from writing to the same
- * file.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat Segment Info}</td>
- * <td>.si</td>
- * <td>Stores metadata about a segment</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat Compound File}</td>
- * <td>.cfs, .cfe</td>
- * <td>An optional "virtual" file consisting of all the other index files for
- * systems that frequently run out of file handles.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Fields}</td>
- * <td>.fnm</td>
- * <td>Stores information about the fields</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Index}</td>
- * <td>.fdx</td>
- * <td>Contains pointers to field data</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Data}</td>
- * <td>.fdt</td>
- * <td>The stored fields for documents</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Dictionary}</td>
- * <td>.tim</td>
- * <td>The term dictionary, stores term info</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Index}</td>
- * <td>.tip</td>
- * <td>The index into the Term Dictionary</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Frequencies}</td>
- * <td>.doc</td>
- * <td>Contains the list of docs which contain each term along with frequency</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Positions}</td>
- * <td>.pos</td>
- * <td>Stores position information about where a term occurs in the index</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Payloads}</td>
- * <td>.pay</td>
- * <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Norms}</td>
- * <td>.nvd, .nvm</td>
- * <td>Encodes length and boost factors for docs and fields</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-Document Values}</td>
- * <td>.dvd, .dvm</td>
- * <td>Encodes additional scoring factors or other per-document information.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Index}</td>
- * <td>.tvx</td>
- * <td>Stores offset into the document data file</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Data}</td>
- * <td>.tvd</td>
- * <td>Contains term vector data.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live Documents}</td>
- * <td>.liv</td>
- * <td>Info about what documents are live</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}</td>
- * <td>.kdd, .kdi, .kdm</td>
- * <td>Holds indexed points</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}</td>
- * <td>.vec, .vem, .veq, .vex, .veb, .vemb</td>
- * <td>Holds indexed vectors; <code>.vec</code> files contain the raw vector data,
- * <code>.vem</code> the vector metadata, <code>.veq</code> the quantized vector data, <code>.vex</code> the
- * hnsw graph data, <code>.veb</code>the binary quantized vector data, and
- * <code>.vemb</code>the binary quantized vector metadata</td>
- * </tr>
- * </table>
- *
- * </div> <a id="Lock_File"></a>
- *
- * <h3>Lock File</h3>
- *
- * The write lock, which is stored in the index directory by default, is named "write.lock". If the
- * lock directory is different from the index directory then the write lock will be named
- * "XXXX-write.lock" where XXXX is a unique prefix derived from the full path to the index
- * directory. When this file is present, a writer is currently modifying the index (adding or
- * removing documents). This lock file ensures that only one writer is modifying the index at a
- * time. <a id="History"></a>
- *
- * <h3>History</h3>
- *
- * <p>Compatibility notes are provided in this document, describing how file formats have changed
- * from prior versions:
- *
- * <ul>
- * <li>In version 2.1, the file format was changed to allow lock-less commits (ie, no more commit
- * lock). The change is fully backwards compatible: you can open a pre-2.1 index for searching
- * or adding/deleting of docs. When the new segments file is saved (committed), it will be
- * written in the new file format (meaning no specific "upgrade" process is needed). But note
- * that once a commit has occurred, pre-2.1 Lucene will not be able to read the index.
- * <li>In version 2.3, the file format was changed to allow segments to share a single set of doc
- * store (vectors & stored fields) files. This allows for faster indexing in certain
- * cases. The change is fully backwards compatible (in the same way as the lock-less commits
- * change in 2.1).
- * <li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not Java's modified
- * UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">LUCENE-510</a> for
- * details.
- * <li>In version 2.9, an optional opaque Map<String,String> CommitUserData may be passed to
- * IndexWriter's commit methods (and later retrieved), which is recorded in the segments_N
- * file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">LUCENE-1382</a> for
- * details. Also, diagnostics were added to each segment written recording details about why
- * it was written (due to flush, merge; which OS/JRE was used; etc.). See issue <a
- * href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.
- * <li>In version 3.0, compressed fields are no longer written to the index (they can still be
- * read, but on merge the new segment will write them, uncompressed). See issue <a
- * href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a> for details.
- * <li>In version 3.1, segments records the code version that created them. See <a
- * href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.
- * Additionally segments track explicitly whether or not they have term vectors. See <a
- * href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a> for details.
- * <li>In version 3.2, numeric fields are written as natively to stored fields file, previously
- * they were stored in text format only.
- * <li>In version 3.4, fields can omit position data while still indexing term frequencies.
- * <li>In version 4.0, the format of the inverted index became extensible via the {@link
- * org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage ({@code DocValues})
- * was introduced. Normalization factors need no longer be a single byte, they can be any
- * {@link org.apache.lucene.index.NumericDocValues NumericDocValues}. Terms need not be
- * unicode strings, they can be any byte sequence. Term offsets can optionally be indexed into
- * the postings lists. Payloads can be stored in the term vectors.
- * <li>In version 4.1, the format of the postings list changed to use either of FOR compression or
- * variable-byte encoding, depending upon the frequency of the term. Terms appearing only once
- * were changed to inline directly into the term dictionary. Stored fields are compressed by
- * default.
- * <li>In version 4.2, term vectors are compressed by default. DocValues has a new multi-valued
- * type (SortedSet), that can be used for faceting/grouping/joining on multi-valued fields.
- * <li>In version 4.5, DocValues were extended to explicitly represent missing values.
- * <li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
- * allow updating NumericDocValues fields.
- * <li>In version 4.8, checksum footers were added to the end of each index file for improved data
- * integrity. Specifically, the last 8 bytes of every index file contain the zlib-crc32
- * checksum of the file.
- * <li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric) that is
- * suitable for faceting/sorting/analytics.
- * <li>In version 5.4, DocValues have been improved to store more information on disk: addresses
- * for binary fields and ord indexes for multi-valued fields.
- * <li>In version 6.0, Points were added, for multi-dimensional range/distance search.
- * <li>In version 6.2, new Segment info format that reads/writes the index sort, to support index
- * sorting.
- * <li>In version 7.0, DocValues have been improved to better support sparse doc values thanks to
- * an iterator API.
- * <li>In version 8.0, postings have been enhanced to record, for each block of doc ids, the (term
- * freq, normalization factor) pairs that may trigger the maximum score of the block. This
- * information is recorded alongside skip data in order to be able to skip blocks of doc ids
- * if they may not produce high enough scores. Additionally doc values and norms has been
- * extended with jump-tables to make access O(1) instead of O(n), where n is the number of
- * elements to skip when advancing in the data.
- * <li>In version 8.4, postings, positions, offsets and payload lengths have move to a more
- * performant encoding that is vectorized.
- * <li>In version 8.6, index sort serialization is delegated to the sorts themselves, to allow
- * user-defined sorts to be used
- * <li>In version 8.6, points fields split the index tree and leaf data into separate files, to
- * allow for different access patterns to the different data structures
- * <li>In version 8.7, stored fields compression became adaptive to better handle documents with
- * smaller stored fields.
- * <li>In version 9.0, vector-valued fields were added.
- * <li>In version 9.1, vector-valued fields were modified to add a graph hierarchy.
- * <li>In version 9.2, docs of vector-valued fields were moved from .vem to .vec and encoded by
- * IndexDISI. ordToDoc mappings was added to .vem.
- * <li>In version 9.5, HNSW graph connections were changed to be delta-encoded with vints.
- * Additionally, metadata file size improvements were made by delta-encoding nodes by graph
- * layer and not writing the node ids for the zeroth layer.
- * <li>In version 9.9, Vector scalar quantization support was added. Allowing the HNSW vector
- * format to utilize int8 quantized vectors for float32 vector search.
- * <li>In version 9.12, skip data was refactored to have only two levels: every 128 docs and every
- * 4,06 docs, and to be inlined in postings lists. This resulted in a speedup for queries that
- * need skipping, especially conjunctions.
- * <li>In version 10.1, block encoding changed to be optimized for int[] storage instead of
- * long[].
- * <li>In version 10.2, new vector asymmetric binary quantization that for HNSW and flat formats.
- * </ul>
- *
- * <a id="Limitations"></a>
- *
- * <h3>Limitations</h3>
- *
- * <div>
- *
- * <p>Lucene uses a Java <code>int</code> to refer to document numbers, and the index file format
- * uses an <code>Int32</code> on-disk to store document numbers. This is a limitation of both the
- * index file format and the current implementation. Eventually these should be replaced with either
- * <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
- * VInt} values which have no limit. </div>
- */
-package org.apache.lucene.codecs.lucene102;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
index b28e0c93c89..3f199db83be 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
@@ -105,6 +105,18 @@ public class Lucene104HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat
HNSW_GRAPH_THRESHOLD);
}
+ /**
+ * Constructs a format using the given graph construction parameters.
+ *
+ * @param encoding the quantization encoding used to encode the vectors
+ * @param maxConn the maximum number of connections to a node in the HNSW graph
+ * @param beamWidth the size of the queue maintained during graph construction.
+ */
+ public Lucene104HnswScalarQuantizedVectorsFormat(
+ ScalarEncoding encoding, int maxConn, int beamWidth) {
+ this(encoding, maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null, HNSW_GRAPH_THRESHOLD);
+ }
+
/**
* Constructs a format using the given graph construction parameters and scalar quantization.
*
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
index c614014dec6..3ac106d11c8 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
@@ -14,7 +14,5 @@
# limitations under the License.
org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat
-org.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat
-org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat
org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat
org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat