You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ch...@apache.org on 2024/05/31 14:02:00 UTC
(lucene) branch branch_9_11 updated: SimpleText[Float|Byte]VectorValues::scorer should return null when the vector values is empty (#13444)
This is an automated email from the ASF dual-hosted git repository.
chegar pushed a commit to branch branch_9_11
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9_11 by this push:
new 46d4872d0f5 SimpleText[Float|Byte]VectorValues::scorer should return null when the vector values is empty (#13444)
46d4872d0f5 is described below
commit 46d4872d0f550fe20be1f5da1474b33dfaaedd6c
Author: Chris Hegarty <62...@users.noreply.github.com>
AuthorDate: Fri May 31 14:27:25 2024 +0100
SimpleText[Float|Byte]VectorValues::scorer should return null when the vector values is empty (#13444)
This commit ensures that SimpleText[Float|Byte]VectorValues::scorer returns null when the vector values is empty, as per the scorer javadoc. Other KnnVectorsReader implementations have specialised empty implementations that do similar, e.g. OffHeapFloatVectorValues.EmptyOffHeapVectorValues. The VectorScorer interface in new in Lucene 9.11, see #13181
An existing test randomly hits this, but a new test has been added that exercises this code path consistently. It's also useful to verify other KnnVectorsReader implementations.
---
.../lucene90/Lucene90HnswVectorsReader.java | 3 ++
.../lucene91/Lucene91HnswVectorsReader.java | 3 ++
.../lucene90/TestLucene90HnswVectorsFormat.java | 5 ++
.../lucene91/TestLucene91HnswVectorsFormat.java | 5 ++
.../lucene92/TestLucene92HnswVectorsFormat.java | 5 ++
.../simpletext/SimpleTextKnnVectorsReader.java | 6 +++
.../tests/index/BaseKnnVectorsFormatTestCase.java | 53 ++++++++++++++++++++++
7 files changed, 80 insertions(+)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
index b277275fe05..5376de7b971 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java
@@ -451,6 +451,9 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {
@Override
public VectorScorer scorer(float[] target) {
+ if (size() == 0) {
+ return null;
+ }
OffHeapFloatVectorValues values = this.copy();
return new VectorScorer() {
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
index 2ebb6382da5..2415cc53f58 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java
@@ -497,6 +497,9 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
@Override
public VectorScorer scorer(float[] target) {
+ if (size == 0) {
+ return null;
+ }
OffHeapFloatVectorValues values = this.copy();
return new VectorScorer() {
@Override
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java
index b914acf3fbb..720e1f56468 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java
@@ -73,4 +73,9 @@ public class TestLucene90HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
public void testByteVectorScorerIteration() {
// unimplemented
}
+
+ @Override
+ public void testEmptyByteVectorData() {
+ // unimplemented
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java
index b27a42700cb..09b5a50b4bc 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java
@@ -72,4 +72,9 @@ public class TestLucene91HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
public void testByteVectorScorerIteration() {
// unimplemented
}
+
+ @Override
+ public void testEmptyByteVectorData() {
+ // unimplemented
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java
index aaee5abe4ad..5189791ef17 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java
@@ -62,4 +62,9 @@ public class TestLucene92HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
public void testByteVectorScorerIteration() {
// unimplemented
}
+
+ @Override
+ public void testEmptyByteVectorData() {
+ // unimplemented
+ }
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
index bcd52a39c04..66fc9af07ff 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
@@ -395,6 +395,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
@Override
public VectorScorer scorer(float[] target) {
+ if (size() == 0) {
+ return null;
+ }
SimpleTextFloatVectorValues simpleTextFloatVectorValues =
new SimpleTextFloatVectorValues(this);
return new VectorScorer() {
@@ -504,6 +507,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
@Override
public VectorScorer scorer(byte[] target) {
+ if (size() == 0) {
+ return null;
+ }
SimpleTextByteVectorValues simpleTextByteVectorValues = new SimpleTextByteVectorValues(this);
return new VectorScorer() {
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index f9daae8ebbb..ccceba42c98 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.tests.index;
+import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.io.ByteArrayOutputStream;
@@ -847,6 +848,58 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
}
}
+ public void testEmptyFloatVectorData() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
+ var doc1 = new Document();
+ doc1.add(new StringField("id", "0", Field.Store.NO));
+ doc1.add(new KnnFloatVectorField("v", new float[] {2, 3, 5, 6}, DOT_PRODUCT));
+ w.addDocument(doc1);
+
+ var doc2 = new Document();
+ doc2.add(new StringField("id", "1", Field.Store.NO));
+ w.addDocument(doc2);
+
+ w.deleteDocuments(new Term("id", Integer.toString(0)));
+ w.commit();
+ w.forceMerge(1);
+
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader r = getOnlyLeafReader(reader);
+ FloatVectorValues values = r.getFloatVectorValues("v");
+ assertNotNull(values);
+ assertEquals(0, values.size());
+ assertNull(values.scorer(new float[] {2, 3, 5, 6}));
+ }
+ }
+ }
+
+ public void testEmptyByteVectorData() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
+ var doc1 = new Document();
+ doc1.add(new StringField("id", "0", Field.Store.NO));
+ doc1.add(new KnnByteVectorField("v", new byte[] {2, 3, 5, 6}, DOT_PRODUCT));
+ w.addDocument(doc1);
+
+ var doc2 = new Document();
+ doc2.add(new StringField("id", "1", Field.Store.NO));
+ w.addDocument(doc2);
+
+ w.deleteDocuments(new Term("id", Integer.toString(0)));
+ w.commit();
+ w.forceMerge(1);
+
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReader r = getOnlyLeafReader(reader);
+ ByteVectorValues values = r.getByteVectorValues("v");
+ assertNotNull(values);
+ assertEquals(0, values.size());
+ assertNull(values.scorer(new byte[] {2, 3, 5, 6}));
+ }
+ }
+ }
+
protected VectorSimilarityFunction randomSimilarity() {
return VectorSimilarityFunction.values()[
random().nextInt(VectorSimilarityFunction.values().length)];