You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2024/03/29 13:29:58 UTC
(lucene) branch branch_9x updated: Use FieldInfo vector similarity in knn readers (#13237)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 0629a99f540 Use FieldInfo vector similarity in knn readers (#13237)
0629a99f540 is described below
commit 0629a99f540b6a982b67c5e3e07d153250f60d33
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Fri Mar 29 09:10:39 2024 -0400
Use FieldInfo vector similarity in knn readers (#13237)
Both the KnnWriters & FieldInfo keep track of the vector similarity used by a given field. This commit ensures they are the same and utilizes the FieldInfo one (which, while these are enums, are exactly the same).
---
.../lucene90/Lucene90HnswVectorsReader.java | 15 ++++++++++++---
.../lucene91/Lucene91HnswVectorsReader.java | 15 ++++++++++++---
.../lucene92/Lucene92HnswVectorsReader.java | 15 ++++++++++++---
.../lucene94/Lucene94HnswVectorsReader.java | 15 ++++++++++++---
.../lucene95/Lucene95HnswVectorsReader.java | 15 ++++++++++++---
.../lucene/codecs/lucene99/Lucene99FlatVectorsReader.java | 15 ++++++++++++---
.../lucene/codecs/lucene99/Lucene99HnswVectorsReader.java | 15 ++++++++++++---
.../lucene99/Lucene99ScalarQuantizedVectorsReader.java | 15 ++++++++++++---
8 files changed, 96 insertions(+), 24 deletions(-)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
index d8fff52012e..dbf5f2ef461 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
@@ -158,7 +158,7 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -200,9 +200,18 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
return VectorSimilarityFunction.values()[similarityFunctionId];
}
- private FieldEntry readField(DataInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
index 16dbdc199c8..e378fc83dc1 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
@@ -150,7 +150,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -192,9 +192,18 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
return VectorSimilarityFunction.values()[similarityFunctionId];
}
- private FieldEntry readField(DataInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
index 5b4cdf82578..fb700cf7a30 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsReader.java
@@ -149,7 +149,7 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -191,9 +191,18 @@ public final class Lucene92HnswVectorsReader extends KnnVectorsReader {
return VectorSimilarityFunction.values()[similarityFunctionId];
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
index cbb59865dc2..c52442316f6 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
@@ -152,7 +152,7 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -216,10 +216,19 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
return VectorEncoding.values()[encodingId];
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
index f1fcb3c47d3..9ac52123f35 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsReader.java
@@ -164,7 +164,7 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader implements
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -228,10 +228,19 @@ public final class Lucene95HnswVectorsReader extends KnnVectorsReader implements
return VectorEncoding.values()[encodingId];
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
index 5021b43f713..0285509d300 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
@@ -143,7 +143,7 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -190,10 +190,19 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {
}
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
index 5a72b8544d8..65c35e94687 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java
@@ -153,7 +153,7 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -200,10 +200,19 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader
return VectorEncoding.values()[encodingId];
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index 7643e1396bf..cd3f7f19b3f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -108,7 +108,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
if (info == null) {
throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
}
- FieldEntry fieldEntry = readField(meta);
+ FieldEntry fieldEntry = readField(meta, info);
validateFieldEntry(info, fieldEntry);
fields.put(info.name, fieldEntry);
}
@@ -236,10 +236,19 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
return size;
}
- private FieldEntry readField(IndexInput input) throws IOException {
+ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException {
VectorEncoding vectorEncoding = readVectorEncoding(input);
VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
- return new FieldEntry(input, vectorEncoding, similarityFunction);
+ if (similarityFunction != info.getVectorSimilarityFunction()) {
+ throw new IllegalStateException(
+ "Inconsistent vector similarity function for field=\""
+ + info.name
+ + "\"; "
+ + similarityFunction
+ + " != "
+ + info.getVectorSimilarityFunction());
+ }
+ return new FieldEntry(input, vectorEncoding, info.getVectorSimilarityFunction());
}
@Override