You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ch...@apache.org on 2024/05/16 14:00:01 UTC
(lucene) branch main updated: Fix bug in SQ when just a single vector present in a segment (#13374)
This is an automated email from the ASF dual-hosted git repository.
chegar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 3d671a0fbef Fix bug in SQ when just a single vector present in a segment (#13374)
3d671a0fbef is described below
commit 3d671a0fbef159e970b060d3f942fba481bafc8b
Author: Chris Hegarty <62...@users.noreply.github.com>
AuthorDate: Thu May 16 14:59:56 2024 +0100
Fix bug in SQ when just a single vector present in a segment (#13374)
This commit fixes a corner case in the ScalarQuantizer when just a single vector is present. I ran into this when updating a test that previously passed successfully with Lucene 9.10 but fails in 9.x.
The score error correction is calculated to be NaN, as there are no score docs or variance.
---
lucene/CHANGES.txt | 2 +
.../lucene/util/quantization/ScalarQuantizer.java | 2 +-
.../TestLucene99ScalarQuantizedVectorScorer.java | 55 ++++++++++++++++++++++
3 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a00c594f9ac..8afa946ca72 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -372,6 +372,8 @@ Bug Fixes
* GITHUB#13378: Fix points writing with no values (Chris Hegarty)
+* GITHUB#13374: Fix bug in SQ when just a single vector present in a segment (Chris Hegarty)
+
Build
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
index 421c181f021..fb07e005571 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
@@ -698,7 +698,7 @@ public class ScalarQuantizer {
}
corr.add(1 - errors.var() / scoreVariance);
}
- return corr.mean;
+ return Double.isNaN(corr.mean) ? 0.0 : corr.mean;
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
index cbfecdeb1da..a244302692e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
@@ -29,12 +29,14 @@ import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -252,6 +254,59 @@ public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {
}
}
+ public void testSingleVectorPerSegmentCosine() throws IOException {
+ testSingleVectorPerSegment(VectorSimilarityFunction.COSINE);
+ }
+
+ public void testSingleVectorPerSegmentDot() throws IOException {
+ testSingleVectorPerSegment(VectorSimilarityFunction.DOT_PRODUCT);
+ }
+
+ public void testSingleVectorPerSegmentEuclidean() throws IOException {
+ testSingleVectorPerSegment(VectorSimilarityFunction.EUCLIDEAN);
+ }
+
+ public void testSingleVectorPerSegmentMIP() throws IOException {
+ testSingleVectorPerSegment(VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT);
+ }
+
+ private void testSingleVectorPerSegment(VectorSimilarityFunction sim) throws IOException {
+ var codec = getCodec(7, false);
+ try (Directory dir = newDirectory()) {
+ try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig().setCodec(codec))) {
+ Document doc2 = new Document();
+ doc2.add(new KnnFloatVectorField("field", new float[] {0.8f, 0.6f}, sim));
+ doc2.add(newTextField("id", "A", Field.Store.YES));
+ writer.addDocument(doc2);
+ writer.commit();
+
+ Document doc1 = new Document();
+ doc1.add(new KnnFloatVectorField("field", new float[] {0.6f, 0.8f}, sim));
+ doc1.add(newTextField("id", "B", Field.Store.YES));
+ writer.addDocument(doc1);
+ writer.commit();
+
+ Document doc3 = new Document();
+ doc3.add(new KnnFloatVectorField("field", new float[] {-0.6f, -0.8f}, sim));
+ doc3.add(newTextField("id", "C", Field.Store.YES));
+ writer.addDocument(doc3);
+ writer.commit();
+
+ writer.forceMerge(1);
+ }
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ LeafReader leafReader = getOnlyLeafReader(reader);
+ StoredFields storedFields = reader.storedFields();
+ float[] queryVector = new float[] {0.6f, 0.8f};
+ var hits = leafReader.searchNearestVectors("field", queryVector, 3, null, 100);
+ assertEquals(hits.scoreDocs.length, 3);
+ assertEquals("B", storedFields.document(hits.scoreDocs[0].doc).get("id"));
+ assertEquals("A", storedFields.document(hits.scoreDocs[1].doc).get("id"));
+ assertEquals("C", storedFields.document(hits.scoreDocs[2].doc).get("id"));
+ }
+ }
+ }
+
private static byte[] floatToByteArray(float value) {
return ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putFloat(value).array();
}