You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ch...@apache.org on 2024/05/16 14:00:01 UTC

(lucene) branch main updated: Fix bug in SQ when just a single vector present in a segment (#13374)

This is an automated email from the ASF dual-hosted git repository.

chegar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new 3d671a0fbef Fix bug in SQ when just a single vector present in a segment (#13374)
3d671a0fbef is described below

commit 3d671a0fbef159e970b060d3f942fba481bafc8b
Author: Chris Hegarty <62...@users.noreply.github.com>
AuthorDate: Thu May 16 14:59:56 2024 +0100

    Fix bug in SQ when just a single vector present in a segment (#13374)
    
    This commit fixes a corner case in the ScalarQuantizer when just a single vector is present. I ran into this when updating a test that previously passed successfully with Lucene 9.10 but fails in 9.x.
    
    The score error correction is calculated to be NaN, as there are no score docs or variance.
---
 lucene/CHANGES.txt                                 |  2 +
 .../lucene/util/quantization/ScalarQuantizer.java  |  2 +-
 .../TestLucene99ScalarQuantizedVectorScorer.java   | 55 ++++++++++++++++++++++
 3 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a00c594f9ac..8afa946ca72 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -372,6 +372,8 @@ Bug Fixes
 
 * GITHUB#13378: Fix points writing with no values (Chris Hegarty)
 
+* GITHUB#13374: Fix bug in SQ when just a single vector present in a segment (Chris Hegarty)
+
 Build
 ---------------------
 
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
index 421c181f021..fb07e005571 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
@@ -698,7 +698,7 @@ public class ScalarQuantizer {
         }
         corr.add(1 - errors.var() / scoreVariance);
       }
-      return corr.mean;
+      return Double.isNaN(corr.mean) ? 0.0 : corr.mean;
     }
   }
 }
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
index cbfecdeb1da..a244302692e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
@@ -29,12 +29,14 @@ import org.apache.lucene.codecs.KnnVectorsReader;
 import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.index.CodecReader;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -252,6 +254,59 @@ public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {
     }
   }
 
+  public void testSingleVectorPerSegmentCosine() throws IOException {
+    testSingleVectorPerSegment(VectorSimilarityFunction.COSINE);
+  }
+
+  public void testSingleVectorPerSegmentDot() throws IOException {
+    testSingleVectorPerSegment(VectorSimilarityFunction.DOT_PRODUCT);
+  }
+
+  public void testSingleVectorPerSegmentEuclidean() throws IOException {
+    testSingleVectorPerSegment(VectorSimilarityFunction.EUCLIDEAN);
+  }
+
+  public void testSingleVectorPerSegmentMIP() throws IOException {
+    testSingleVectorPerSegment(VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT);
+  }
+
+  private void testSingleVectorPerSegment(VectorSimilarityFunction sim) throws IOException {
+    var codec = getCodec(7, false);
+    try (Directory dir = newDirectory()) {
+      try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig().setCodec(codec))) {
+        Document doc2 = new Document();
+        doc2.add(new KnnFloatVectorField("field", new float[] {0.8f, 0.6f}, sim));
+        doc2.add(newTextField("id", "A", Field.Store.YES));
+        writer.addDocument(doc2);
+        writer.commit();
+
+        Document doc1 = new Document();
+        doc1.add(new KnnFloatVectorField("field", new float[] {0.6f, 0.8f}, sim));
+        doc1.add(newTextField("id", "B", Field.Store.YES));
+        writer.addDocument(doc1);
+        writer.commit();
+
+        Document doc3 = new Document();
+        doc3.add(new KnnFloatVectorField("field", new float[] {-0.6f, -0.8f}, sim));
+        doc3.add(newTextField("id", "C", Field.Store.YES));
+        writer.addDocument(doc3);
+        writer.commit();
+
+        writer.forceMerge(1);
+      }
+      try (DirectoryReader reader = DirectoryReader.open(dir)) {
+        LeafReader leafReader = getOnlyLeafReader(reader);
+        StoredFields storedFields = reader.storedFields();
+        float[] queryVector = new float[] {0.6f, 0.8f};
+        var hits = leafReader.searchNearestVectors("field", queryVector, 3, null, 100);
+        assertEquals(hits.scoreDocs.length, 3);
+        assertEquals("B", storedFields.document(hits.scoreDocs[0].doc).get("id"));
+        assertEquals("A", storedFields.document(hits.scoreDocs[1].doc).get("id"));
+        assertEquals("C", storedFields.document(hits.scoreDocs[2].doc).get("id"));
+      }
+    }
+  }
+
   private static byte[] floatToByteArray(float value) {
     return ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putFloat(value).array();
   }