You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2024/03/29 13:10:43 UTC
(lucene) branch main updated: Use FieldInfo vector similarity in knn readers (#13237)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 69172b14ce8 Use FieldInfo vector similarity in knn readers (#13237)
69172b14ce8 is described below
commit 69172b14ce88a0c0784e16add44f2f504bc4038e
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Fri Mar 29 09:10:39 2024 -0400
Use FieldInfo vector similarity in knn readers (#13237)
Both the KnnWriters & FieldInfo keep track of the vector similarity used by a given field. This commit ensures they are the same and utilizes the FieldInfo one (which, while these are enums, are exactly the same).
---
.../lucene90/Lucene90HnswVectorsReader.java | 15 ++++++++++++---
.../lucene91/Lucene91HnswVectorsReader.java | 15 ++++++++++++---
.../lucene92/Lucene92HnswVectorsReader.java | 15 ++++++++++++---
.../lucene94/Lucene94HnswVectorsReader.java | 15 ++++++++++++---
.../lucene95/Lucene95HnswVectorsReader.java | 15 ++++++++++++---
.../lucene/codecs/lucene99/Lucene99FlatVectorsReader.java | 15 ++++++++++++---
.../lucene/codecs/lucene99/Lucene99HnswVectorsReader.java | 15 ++++++++++++---
.../lucene99/Lucene99ScalarQuantizedVectorsReader.java | 15 ++++++++++++---
8 files changed, 96 insertions(+), 24 deletions(-)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
index b72224d9b48..d48817f54bd 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
@@ -158,7 +158,7 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -200,9 +200,18 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
return VectorSimilarityFunction.values()[similarityFunctionId];
}
- private FieldEntry readField(DataInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
index 48063d5761f..e0210c9b19a 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
@@ -150,7 +150,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -192,9 +192,18 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
return VectorSimilarityFunction.values()[similarityFunctionId];
}
- private FieldEntry readField(DataInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
index cbf80cc0c97..a7aa241f42d 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
@@ -149,7 +149,7 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -191,9 +191,18 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {
return VectorSimilarityFunction.values()[similarityFunctionId];
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
index 1a54fff5913..5daec83629f 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
@@ -150,7 +150,7 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -208,10 +208,19 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
return VectorEncoding.values()[encodingId];
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
index 9c00c5ef93c..d0063f9da17 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
@@ -162,7 +162,7 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader implements
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -220,10 +220,19 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader implements
return VectorEncoding.values()[encodingId];
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
index 365883f18b8..81ea9b07023 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
@@ -143,7 +143,7 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -183,10 +183,19 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {
}
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
index efb51c963e0..198740cb06f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
@@ -153,7 +153,7 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -200,10 +200,19 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader
return VectorEncoding.values()[encodingId];
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index f85f21dd62a..96bef956a3f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -108,7 +108,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -236,10 +236,19 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
return size;
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override