You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2023/07/27 18:50:39 UTC

[lucene] branch main updated: Move max vector dims limit to Codec (#12436)

This is an automated email from the ASF dual-hosted git repository.

mayya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new 98320d76164 Move max vector dims limit to Codec (#12436)
98320d76164 is described below

commit 98320d761641bf9e4c0a2c6d05c262b17d415659
Author: Mayya Sharipova <ma...@elastic.co>
AuthorDate: Thu Jul 27 14:50:33 2023 -0400

    Move max vector dims limit to Codec (#12436)
    
    Move vector max dimension limits enforcement into the default Codec's
    KnnVectorsFormat implementation. This allows different implementation
    of knn search algorithms define their own limits of a maximum
    vector dimensions that they can handle.
    
    Closes #12309
---
 lucene/CHANGES.txt                                 |  2 +
 .../org/apache/lucene/codecs/KnnVectorsFormat.java | 16 +++++
 .../codecs/lucene95/Lucene95HnswVectorsFormat.java |  5 ++
 .../codecs/perfield/PerFieldKnnVectorsFormat.java  |  5 ++
 .../java/org/apache/lucene/document/FieldType.java |  8 ---
 .../apache/lucene/document/KnnByteVectorField.java |  4 --
 .../lucene/document/KnnFloatVectorField.java       |  4 --
 .../org/apache/lucene/index/ByteVectorValues.java  |  3 -
 .../org/apache/lucene/index/FloatVectorValues.java |  3 -
 .../org/apache/lucene/index/IndexingChain.java     | 20 ++++++
 .../perfield/TestPerFieldKnnVectorsFormat.java     | 72 ++++++++++++++++++++++
 .../tests/index/BaseFieldInfoFormatTestCase.java   | 11 ++--
 .../tests/index/BaseKnnVectorsFormatTestCase.java  | 55 +++++++++++------
 13 files changed, 164 insertions(+), 44 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a33fa2e86c4..e4b81d05ca0 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -129,6 +129,8 @@ API Changes
 * GITHUB#11248: IntBlockPool's SliceReader, SliceWriter, and all int slice functionality are moved out to MemoryIndex.
   (Stefan Vodita)
 
+* GITHUB#12436: Move max vector dims limit to Codec (Mayya Sharipova)
+
 New Features
 ---------------------
 
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java
index 66623bacadc..a17d844d811 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsFormat.java
@@ -32,6 +32,9 @@ import org.apache.lucene.util.NamedSPILoader;
  */
 public abstract class KnnVectorsFormat implements NamedSPILoader.NamedSPI {
 
+  /** The maximum number of vector dimensions */
+  public static final int DEFAULT_MAX_DIMENSIONS = 1024;
+
   /**
    * This static holder class prevents classloading deadlock by delaying init of doc values formats
    * until needed.
@@ -76,6 +79,19 @@ public abstract class KnnVectorsFormat implements NamedSPILoader.NamedSPI {
   /** Returns a {@link KnnVectorsReader} to read the vectors from the index. */
   public abstract KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException;
 
+  /**
+   * Returns the maximum number of vector dimensions supported by this codec for the given field
+   * name
+   *
+   * <p>Codecs should override this method to specify the maximum number of dimensions they support.
+   *
+   * @param fieldName the field name
+   * @return the maximum number of vector dimensions.
+   */
+  public int getMaxDimensions(String fieldName) {
+    return DEFAULT_MAX_DIMENSIONS;
+  }
+
   /**
    * EMPTY throws an exception when written. It acts as a sentinel indicating a Codec that does not
    * support vectors.
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsFormat.java
index cb3e5ef8b10..f1594b3068c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsFormat.java
@@ -185,6 +185,11 @@ public final class Lucene95HnswVectorsFormat extends KnnVectorsFormat {
     return new Lucene95HnswVectorsReader(state);
   }
 
+  @Override
+  public int getMaxDimensions(String fieldName) {
+    return 1024;
+  }
+
   @Override
   public String toString() {
     return "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn="
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java
index 1247dff556f..6d344629f1b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java
@@ -80,6 +80,11 @@ public abstract class PerFieldKnnVectorsFormat extends KnnVectorsFormat {
     return new FieldsReader(state);
   }
 
+  @Override
+  public int getMaxDimensions(String fieldName) {
+    return getKnnVectorsFormatForField(fieldName).getMaxDimensions(fieldName);
+  }
+
   /**
    * Returns the numeric vector format that should be used for writing new segments of <code>field
    * </code>.
diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.java
index aba3fa3c3bb..5b37955cc14 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java
@@ -21,7 +21,6 @@ import java.util.Map;
 import java.util.Objects;
 import org.apache.lucene.analysis.Analyzer; // javadocs
 import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.index.PointValues;
@@ -378,13 +377,6 @@ public class FieldType implements IndexableFieldType {
     if (numDimensions <= 0) {
       throw new IllegalArgumentException("vector numDimensions must be > 0; got " + numDimensions);
     }
-    if (numDimensions > FloatVectorValues.MAX_DIMENSIONS) {
-      throw new IllegalArgumentException(
-          "vector numDimensions must be <= FloatVectorValues.MAX_DIMENSIONS (="
-              + FloatVectorValues.MAX_DIMENSIONS
-              + "); got "
-              + numDimensions);
-    }
     this.vectorDimension = numDimensions;
     this.vectorSimilarityFunction = Objects.requireNonNull(similarity);
     this.vectorEncoding = Objects.requireNonNull(encoding);
diff --git a/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java b/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java
index 87cb6a9f056..ddeef44c72a 100644
--- a/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/KnnByteVectorField.java
@@ -46,10 +46,6 @@ public class KnnByteVectorField extends Field {
     if (dimension == 0) {
       throw new IllegalArgumentException("cannot index an empty vector");
     }
-    if (dimension > ByteVectorValues.MAX_DIMENSIONS) {
-      throw new IllegalArgumentException(
-          "cannot index vectors with dimension greater than " + ByteVectorValues.MAX_DIMENSIONS);
-    }
     if (similarityFunction == null) {
       throw new IllegalArgumentException("similarity function must not be null");
     }
diff --git a/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java b/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java
index 9d1cd02c013..63a55ddc669 100644
--- a/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/KnnFloatVectorField.java
@@ -47,10 +47,6 @@ public class KnnFloatVectorField extends Field {
     if (dimension == 0) {
       throw new IllegalArgumentException("cannot index an empty vector");
     }
-    if (dimension > FloatVectorValues.MAX_DIMENSIONS) {
-      throw new IllegalArgumentException(
-          "cannot index vectors with dimension greater than " + FloatVectorValues.MAX_DIMENSIONS);
-    }
     if (similarityFunction == null) {
       throw new IllegalArgumentException("similarity function must not be null");
     }
diff --git a/lucene/core/src/java/org/apache/lucene/index/ByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/index/ByteVectorValues.java
index 5d532a7f9e8..e731e727aa8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ByteVectorValues.java
@@ -28,9 +28,6 @@ import org.apache.lucene.search.DocIdSetIterator;
  */
 public abstract class ByteVectorValues extends DocIdSetIterator {
 
-  /** The maximum length of a vector */
-  public static final int MAX_DIMENSIONS = 1024;
-
   /** Sole constructor */
   protected ByteVectorValues() {}
 
diff --git a/lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java
index 9de6b57531e..0c3194bfac8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FloatVectorValues.java
@@ -28,9 +28,6 @@ import org.apache.lucene.search.DocIdSetIterator;
  */
 public abstract class FloatVectorValues extends DocIdSetIterator {
 
-  /** The maximum length of a vector */
-  public static final int MAX_DIMENSIONS = 1024;
-
   /** Sole constructor */
   protected FloatVectorValues() {}
 
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index 9a8b264a86d..4d9167d93a0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -621,6 +621,12 @@ final class IndexingChain implements Accountable {
       final Sort indexSort = indexWriterConfig.getIndexSort();
       validateIndexSortDVType(indexSort, pf.fieldName, s.docValuesType);
     }
+    if (s.vectorDimension != 0) {
+      validateMaxVectorDimension(
+          pf.fieldName,
+          s.vectorDimension,
+          indexWriterConfig.getCodec().knnVectorsFormat().getMaxDimensions(pf.fieldName));
+    }
     FieldInfo fi =
         fieldInfos.add(
             new FieldInfo(
@@ -831,6 +837,20 @@ final class IndexingChain implements Accountable {
     }
   }
 
+  private static void validateMaxVectorDimension(
+      String fieldName, int vectorDim, int maxVectorDim) {
+    if (vectorDim > maxVectorDim) {
+      throw new IllegalArgumentException(
+          "Field ["
+              + fieldName
+              + "]"
+              + "vector's dimensions must be <= ["
+              + maxVectorDim
+              + "]; got "
+              + vectorDim);
+    }
+  }
+
   private void validateIndexSortDVType(Sort indexSort, String fieldToValidate, DocValuesType dvType)
       throws IOException {
     for (SortField sortField : indexSort.getSort()) {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java
index c435de1047d..dd06dd6c061 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java
@@ -29,6 +29,7 @@ import org.apache.lucene.codecs.KnnFieldVectorsWriter;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.KnnVectorsReader;
 import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.KnnFloatVectorField;
@@ -43,6 +44,9 @@ import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
 import org.apache.lucene.index.Sorter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.KnnFloatVectorQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -162,6 +166,50 @@ public class TestPerFieldKnnVectorsFormat extends BaseKnnVectorsFormatTestCase {
     }
   }
 
+  public void testMaxDimensionsPerFieldFormat() throws IOException {
+    try (Directory directory = newDirectory()) {
+      IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
+      KnnVectorsFormat format1 =
+          new KnnVectorsFormatMaxDims32(new Lucene95HnswVectorsFormat(16, 100));
+      KnnVectorsFormat format2 = new Lucene95HnswVectorsFormat(16, 100);
+      iwc.setCodec(
+          new AssertingCodec() {
+            @Override
+            public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+              if ("field1".equals(field)) {
+                return format1;
+              } else {
+                return format2;
+              }
+            }
+          });
+      try (IndexWriter writer = new IndexWriter(directory, iwc)) {
+        Document doc1 = new Document();
+        doc1.add(new KnnFloatVectorField("field1", new float[33]));
+        Exception exc =
+            expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc1));
+        assertTrue(exc.getMessage().contains("vector's dimensions must be <= [32]"));
+
+        Document doc2 = new Document();
+        doc2.add(new KnnFloatVectorField("field1", new float[32]));
+        doc2.add(new KnnFloatVectorField("field2", new float[33]));
+        writer.addDocument(doc2);
+      }
+
+      // Check that the vectors were written
+      try (IndexReader reader = DirectoryReader.open(directory)) {
+        IndexSearcher searcher = new IndexSearcher(reader);
+        Query query1 = new KnnFloatVectorQuery("field1", new float[32], 10);
+        TopDocs topDocs1 = searcher.search(query1, 1);
+        assertEquals(1, topDocs1.scoreDocs.length);
+
+        Query query2 = new KnnFloatVectorQuery("field2", new float[33], 10);
+        TopDocs topDocs2 = searcher.search(query2, 1);
+        assertEquals(1, topDocs2.scoreDocs.length);
+      }
+    }
+  }
+
   private static class WriteRecordingKnnVectorsFormat extends KnnVectorsFormat {
     private final KnnVectorsFormat delegate;
     private final Set<String> fieldsWritten;
@@ -216,4 +264,28 @@ public class TestPerFieldKnnVectorsFormat extends BaseKnnVectorsFormatTestCase {
       return delegate.fieldsReader(state);
     }
   }
+
+  private static class KnnVectorsFormatMaxDims32 extends KnnVectorsFormat {
+    private final KnnVectorsFormat delegate;
+
+    public KnnVectorsFormatMaxDims32(KnnVectorsFormat delegate) {
+      super(delegate.getName());
+      this.delegate = delegate;
+    }
+
+    @Override
+    public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+      return delegate.fieldsWriter(state);
+    }
+
+    @Override
+    public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
+      return delegate.fieldsReader(state);
+    }
+
+    @Override
+    public int getMaxDimensions(String fieldName) {
+      return 32;
+    }
+  }
 }
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java
index f4f3d29a655..7a054c290c3 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java
@@ -32,7 +32,6 @@ import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.FloatVectorValues;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.index.PointValues;
@@ -280,7 +279,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
     var builder = INDEX_PACKAGE_ACCESS.newFieldInfosBuilder(softDeletesField);
 
     for (String field : fieldNames) {
-      IndexableFieldType fieldType = randomFieldType(random());
+      IndexableFieldType fieldType = randomFieldType(random(), field);
       boolean storeTermVectors = false;
       boolean storePayloads = false;
       boolean omitNorms = false;
@@ -319,7 +318,11 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
     dir.close();
   }
 
-  private IndexableFieldType randomFieldType(Random r) {
+  private int getVectorsMaxDimensions(String fieldName) {
+    return Codec.getDefault().knnVectorsFormat().getMaxDimensions(fieldName);
+  }
+
+  private IndexableFieldType randomFieldType(Random r, String fieldName) {
     FieldType type = new FieldType();
 
     if (r.nextBoolean()) {
@@ -352,7 +355,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
     }
 
     if (r.nextBoolean()) {
-      int dimension = 1 + r.nextInt(FloatVectorValues.MAX_DIMENSIONS);
+      int dimension = 1 + r.nextInt(getVectorsMaxDimensions(fieldName));
       VectorSimilarityFunction similarityFunction =
           RandomPicks.randomFrom(r, VectorSimilarityFunction.values());
       VectorEncoding encoding = RandomPicks.randomFrom(r, VectorEncoding.values());
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index 623af11f4e7..20f57663228 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -27,7 +27,6 @@ import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.KnnVectorsReader;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.KnnByteVectorField;
 import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.document.NumericDocValuesField;
@@ -86,6 +85,10 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
     }
   }
 
+  private int getVectorsMaxDimensions(String fieldName) {
+    return Codec.getDefault().knnVectorsFormat().getMaxDimensions(fieldName);
+  }
+
   public void testFieldConstructor() {
     float[] v = new float[1];
     KnnFloatVectorField field = new KnnFloatVectorField("f", v);
@@ -101,14 +104,6 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
         IllegalArgumentException.class,
         () -> new KnnFloatVectorField("f", new float[1], (VectorSimilarityFunction) null));
     expectThrows(IllegalArgumentException.class, () -> new KnnFloatVectorField("f", new float[0]));
-    expectThrows(
-        IllegalArgumentException.class,
-        () -> new KnnFloatVectorField("f", new float[FloatVectorValues.MAX_DIMENSIONS + 1]));
-    expectThrows(
-        IllegalArgumentException.class,
-        () ->
-            new KnnFloatVectorField(
-                "f", new float[FloatVectorValues.MAX_DIMENSIONS + 1], (FieldType) null));
   }
 
   public void testFieldSetValue() {
@@ -478,18 +473,42 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
     try (Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
       Document doc = new Document();
-      expectThrows(
-          IllegalArgumentException.class,
-          () ->
-              doc.add(
-                  new KnnFloatVectorField(
-                      "f",
-                      new float[FloatVectorValues.MAX_DIMENSIONS + 1],
-                      VectorSimilarityFunction.DOT_PRODUCT)));
+      doc.add(
+          new KnnFloatVectorField(
+              "f",
+              new float[getVectorsMaxDimensions("f") + 1],
+              VectorSimilarityFunction.DOT_PRODUCT));
+      Exception exc = expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc));
+      assertTrue(
+          exc.getMessage()
+              .contains("vector's dimensions must be <= [" + getVectorsMaxDimensions("f") + "]"));
 
       Document doc2 = new Document();
-      doc2.add(new KnnFloatVectorField("f", new float[1], VectorSimilarityFunction.EUCLIDEAN));
+      doc2.add(new KnnFloatVectorField("f", new float[1], VectorSimilarityFunction.DOT_PRODUCT));
       w.addDocument(doc2);
+
+      Document doc3 = new Document();
+      doc3.add(
+          new KnnFloatVectorField(
+              "f",
+              new float[getVectorsMaxDimensions("f") + 1],
+              VectorSimilarityFunction.DOT_PRODUCT));
+      exc = expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc3));
+      assertTrue(
+          exc.getMessage()
+              .contains("Inconsistency of field data structures across documents for field [f]"));
+      w.flush();
+
+      Document doc4 = new Document();
+      doc4.add(
+          new KnnFloatVectorField(
+              "f",
+              new float[getVectorsMaxDimensions("f") + 1],
+              VectorSimilarityFunction.DOT_PRODUCT));
+      exc = expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc4));
+      assertTrue(
+          exc.getMessage()
+              .contains("vector's dimensions must be <= [" + getVectorsMaxDimensions("f") + "]"));
     }
   }