You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2024/05/13 17:08:11 UTC
(lucene) branch branch_9x updated: Fix vector scorer interface consistency (#13365)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 8580d50dbb9 Fix vector scorer interface consistency (#13365)
8580d50dbb9 is described below
commit 8580d50dbb91380519a9a7a01ef314ac5c080011
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Mon May 13 13:02:38 2024 -0400
Fix vector scorer interface consistency (#13365)
Follow up to: #13181
I noticed the quantized interface had a slightly different name.
Additionally, testing showed we are inconsistent when there aren't any vectors to score. This makes the response consistent (e.g. null when there aren't any vectors).
---
.../codecs/lucene95/OffHeapByteVectorValues.java | 2 +-
.../codecs/lucene95/OffHeapFloatVectorValues.java | 2 +-
.../Lucene99ScalarQuantizedVectorsReader.java | 2 +-
.../Lucene99ScalarQuantizedVectorsWriter.java | 6 +++---
.../lucene99/OffHeapQuantizedByteVectorValues.java | 8 ++++----
.../org/apache/lucene/index/ByteVectorValues.java | 2 +-
.../org/apache/lucene/index/FloatVectorValues.java | 2 +-
.../search/AbstractVectorSimilarityQuery.java | 3 +++
.../search/FloatVectorSimilarityValuesSource.java | 23 ----------------------
.../quantization/QuantizedByteVectorValues.java | 4 ++--
.../DiversifyingChildrenByteKnnVectorQuery.java | 3 +++
.../DiversifyingChildrenFloatKnnVectorQuery.java | 7 +++++--
.../tests/index/BaseKnnVectorsFormatTestCase.java | 8 ++++++++
13 files changed, 33 insertions(+), 39 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java
index b1f0972f4f9..f45158eadac 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java
@@ -376,7 +376,7 @@ public abstract class OffHeapByteVectorValues extends ByteVectorValues
@Override
public VectorScorer scorer(byte[] query) {
- throw new UnsupportedOperationException();
+ return null;
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java
index 52753325f6b..1f61283b500 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java
@@ -365,7 +365,7 @@ public abstract class OffHeapFloatVectorValues extends FloatVectorValues
@Override
public VectorScorer scorer(float[] query) {
- throw new UnsupportedOperationException();
+ return null;
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index 47125b53795..9fe78a35fc5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -425,7 +425,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
@Override
public VectorScorer scorer(float[] query) throws IOException {
- return quantizedVectorValues.vectorScorer(query);
+ return quantizedVectorValues.scorer(query);
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
index 78e43535470..0e2256a6a4a 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
@@ -1024,7 +1024,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
}
@Override
- public VectorScorer vectorScorer(float[] target) throws IOException {
+ public VectorScorer scorer(float[] target) throws IOException {
throw new UnsupportedOperationException();
}
}
@@ -1097,7 +1097,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
}
@Override
- public VectorScorer vectorScorer(float[] target) throws IOException {
+ public VectorScorer scorer(float[] target) throws IOException {
throw new UnsupportedOperationException();
}
@@ -1203,7 +1203,7 @@ public final class Lucene99ScalarQuantizedVectorsWriter extends FlatVectorsWrite
}
@Override
- public VectorScorer vectorScorer(float[] target) throws IOException {
+ public VectorScorer scorer(float[] target) throws IOException {
throw new UnsupportedOperationException();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java
index 5fee4b1cb08..655dcca1166 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/OffHeapQuantizedByteVectorValues.java
@@ -268,7 +268,7 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVect
}
@Override
- public VectorScorer vectorScorer(float[] target) throws IOException {
+ public VectorScorer scorer(float[] target) throws IOException {
DenseOffHeapVectorValues copy = copy();
RandomVectorScorer vectorScorer =
vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);
@@ -370,7 +370,7 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVect
}
@Override
- public VectorScorer vectorScorer(float[] target) throws IOException {
+ public VectorScorer scorer(float[] target) throws IOException {
SparseOffHeapVectorValues copy = copy();
RandomVectorScorer vectorScorer =
vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target);
@@ -457,8 +457,8 @@ public abstract class OffHeapQuantizedByteVectorValues extends QuantizedByteVect
}
@Override
- public VectorScorer vectorScorer(float[] target) {
- throw new UnsupportedOperationException();
+ public VectorScorer scorer(float[] target) {
+ return null;
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/ByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/index/ByteVectorValues.java
index d04d52b0dcf..d33ca1ca354 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ByteVectorValues.java
@@ -83,7 +83,7 @@ public abstract class ByteVectorValues extends DocIdSetIterator {
* iteration over the scorer will not affect the iteration of this {@link ByteVectorValues}.
*
* @param query the query vector
- * @return a {@link VectorScorer} instance
+ * @return a {@link VectorScorer} instance or null
*/
public abstract VectorScorer scorer(byte[] query) throws IOException;
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java
index 9a5bb31b0c6..e5dbc620f5c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java
@@ -84,7 +84,7 @@ public abstract class FloatVectorValues extends DocIdSetIterator {
* iteration of this {@link FloatVectorValues}.
*
* @param query the query vector
- * @return a {@link VectorScorer} instance
+ * @return a {@link VectorScorer} instance or null
*/
public abstract VectorScorer scorer(float[] query) throws IOException;
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/AbstractVectorSimilarityQuery.java b/lucene/core/src/java/org/apache/lucene/search/AbstractVectorSimilarityQuery.java
index eeab87bba93..458f6d44a6d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AbstractVectorSimilarityQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AbstractVectorSimilarityQuery.java
@@ -260,6 +260,9 @@ abstract class AbstractVectorSimilarityQuery extends Query {
VectorScorer scorer,
DocIdSetIterator acceptDocs,
float threshold) {
+ if (scorer == null) {
+ return null;
+ }
float[] cachedScore = new float[1];
DocIdSetIterator vectorIterator = scorer.iterator();
DocIdSetIterator conjunction =
diff --git a/lucene/core/src/java/org/apache/lucene/search/FloatVectorSimilarityValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/FloatVectorSimilarityValuesSource.java
index 3bf4c0a1887..8a43644cc4e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FloatVectorSimilarityValuesSource.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FloatVectorSimilarityValuesSource.java
@@ -36,29 +36,6 @@ class FloatVectorSimilarityValuesSource extends VectorSimilarityValuesSource {
this.queryVector = vector;
}
- @Override
- public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
- final FloatVectorValues vectorValues = ctx.reader().getFloatVectorValues(fieldName);
- if (vectorValues == null) {
- FloatVectorValues.checkField(ctx.reader(), fieldName);
- return DoubleValues.EMPTY;
- }
- return new DoubleValues() {
- private final VectorScorer scorer = vectorValues.scorer(queryVector);
- private final DocIdSetIterator iterator = scorer.iterator();
-
- @Override
- public double doubleValue() throws IOException {
- return scorer.score();
- }
-
- @Override
- public boolean advanceExact(int doc) throws IOException {
- return doc >= iterator.docID() && (iterator.docID() == doc || iterator.advance(doc) == doc);
- }
- };
- }
-
@Override
public VectorScorer getScorer(LeafReaderContext ctx) throws IOException {
final FloatVectorValues vectorValues = ctx.reader().getFloatVectorValues(fieldName);
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java
index c277abfc634..a0fe957fecb 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/QuantizedByteVectorValues.java
@@ -51,7 +51,7 @@ public abstract class QuantizedByteVectorValues extends DocIdSetIterator {
* Return a {@link VectorScorer} for the given query vector.
*
* @param query the query vector
- * @return a {@link VectorScorer} instance
+ * @return a {@link VectorScorer} instance or null
*/
- public abstract VectorScorer vectorScorer(float[] query) throws IOException;
+ public abstract VectorScorer scorer(float[] query) throws IOException;
}
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.java
index 8c6f0f98470..456a885b49a 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenByteKnnVectorQuery.java
@@ -92,6 +92,9 @@ public class DiversifyingChildrenByteKnnVectorQuery extends KnnByteVectorQuery {
}
VectorScorer scorer = byteVectorValues.scorer(query);
+ if (scorer == null) {
+ return NO_RESULTS;
+ }
DiversifyingChildrenFloatKnnVectorQuery.DiversifyingChildrenVectorScorer vectorScorer =
new DiversifyingChildrenFloatKnnVectorQuery.DiversifyingChildrenVectorScorer(
acceptIterator, parentBitSet, scorer);
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java
index 513da619cae..7b5a656d141 100644
--- a/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java
+++ b/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java
@@ -90,10 +90,13 @@ public class DiversifyingChildrenFloatKnnVectorQuery extends KnnFloatVectorQuery
if (parentBitSet == null) {
return NO_RESULTS;
}
+ VectorScorer floatVectorScorer = floatVectorValues.scorer(query);
+ if (floatVectorScorer == null) {
+ return NO_RESULTS;
+ }
DiversifyingChildrenVectorScorer vectorScorer =
- new DiversifyingChildrenVectorScorer(
- acceptIterator, parentBitSet, floatVectorValues.scorer(query));
+ new DiversifyingChildrenVectorScorer(acceptIterator, parentBitSet, floatVectorScorer);
final int queueSize = Math.min(k, Math.toIntExact(acceptIterator.cost()));
HitQueue queue = new HitQueue(queueSize, true);
TotalHits.Relation relation = TotalHits.Relation.EQUAL_TO;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index ab4157d443f..f9daae8ebbb 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -763,6 +763,10 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
if (vectorValues == null) {
continue;
}
+ if (vectorValues.size() == 0) {
+ assertNull(vectorValues.scorer(vectorToScore));
+ continue;
+ }
VectorScorer scorer = vectorValues.scorer(vectorToScore);
assertNotNull(scorer);
DocIdSetIterator iterator = scorer.iterator();
@@ -818,6 +822,10 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
if (vectorValues == null) {
continue;
}
+ if (vectorValues.size() == 0) {
+ assertNull(vectorValues.scorer(vectorToScore));
+ continue;
+ }
VectorScorer scorer = vectorValues.scorer(vectorToScore);
assertNotNull(scorer);
DocIdSetIterator iterator = scorer.iterator();