You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2023/10/30 13:43:43 UTC
(lucene) branch branch_9x updated: Add back maxConn & beamWidth HNSW codec ctor (#12728)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 24330acf60d Add back maxConn & beamWidth HNSW codec ctor (#12728)
24330acf60d is described below
commit 24330acf60d6fe6fa89160d79b232f427589de57
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Mon Oct 30 09:31:04 2023 -0400
Add back maxConn & beamWidth HNSW codec ctor (#12728)
follow up to https://github.com/apache/lucene/pull/12582
For user convenience, I added back the two parameter ctor for the HNSW codec.
---
.../codecs/lucene99/Lucene99HnswVectorsFormat.java | 10 ++++++++++
.../lucene99/Lucene99ScalarQuantizedVectorsWriter.java | 7 +++++--
.../codecs/lucene99/TestLucene99HnswVectorsFormat.java | 16 +++++++---------
.../codecs/perfield/TestPerFieldKnnVectorsFormat.java | 4 ++--
.../src/test/org/apache/lucene/index/TestKnnGraph.java | 3 +--
.../org/apache/lucene/util/hnsw/HnswGraphTestCase.java | 8 ++++----
.../lucene/tests/codecs/vector/ConfigurableMCodec.java | 4 ++--
7 files changed, 31 insertions(+), 21 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java
index 03a6a4dbd62..038b75e4c48 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java
@@ -191,6 +191,16 @@ public final class Lucene99HnswVectorsFormat extends KnnVectorsFormat {
*
* @param maxConn the maximum number of connections to a node in the HNSW graph
* @param beamWidth the size of the queue maintained during graph construction.
+ */
+ public Lucene99HnswVectorsFormat(int maxConn, int beamWidth) {
+ this(maxConn, beamWidth, null);
+ }
+
+ /**
+ * Constructs a format using the given graph construction parameters and scalar quantization.
+ *
+ * @param maxConn the maximum number of connections to a node in the HNSW graph
+ * @param beamWidth the size of the queue maintained during graph construction.
* @param scalarQuantize the scalar quantization format
* @param numMergeWorkers number of workers (threads) that will be used when doing merge. If
* larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
index 545251edbc5..be9454c14aa 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java
@@ -234,7 +234,8 @@ public final class Lucene99ScalarQuantizedVectorsWriter implements Accountable {
MergedQuantizedVectorValues byteVectorValues =
MergedQuantizedVectorValues.mergeQuantizedByteVectorValues(
fieldInfo, mergeState, mergedQuantizationState);
- writeQuantizedVectorData(tempQuantizedVectorData, byteVectorValues);
+ DocsWithFieldSet docsWithField =
+ writeQuantizedVectorData(tempQuantizedVectorData, byteVectorValues);
CodecUtil.writeFooter(tempQuantizedVectorData);
IOUtils.close(tempQuantizedVectorData);
quantizationDataInput =
@@ -254,7 +255,9 @@ public final class Lucene99ScalarQuantizedVectorsWriter implements Accountable {
fieldInfo.getVectorSimilarityFunction(),
mergedQuantizationState,
new OffHeapQuantizedByteVectorValues.DenseOffHeapVectorValues(
- fieldInfo.getVectorDimension(), byteVectorValues.size(), quantizationDataInput)));
+ fieldInfo.getVectorDimension(),
+ docsWithField.cardinality(),
+ quantizationDataInput)));
} finally {
if (success == false) {
IOUtils.closeWhileHandlingException(quantizationDataInput);
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java
index aa9c9f109e0..8f2fdd2c75f 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java
@@ -33,7 +33,7 @@ public class TestLucene99HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
new FilterCodec("foo", Codec.getDefault()) {
@Override
public KnnVectorsFormat knnVectorsFormat() {
- return new Lucene99HnswVectorsFormat(10, 20, null);
+ return new Lucene99HnswVectorsFormat(10, 20);
}
};
String expectedString =
@@ -42,13 +42,11 @@ public class TestLucene99HnswVectorsFormat extends BaseKnnVectorsFormatTestCase
}
public void testLimits() {
- expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(-1, 20, null));
- expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(0, 20, null));
- expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 0, null));
- expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, -1, null));
- expectThrows(
- IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(512 + 1, 20, null));
- expectThrows(
- IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 3201, null));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(-1, 20));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(0, 20));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 0));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, -1));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(512 + 1, 20));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 3201));
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java
index 792555784c1..fc783f8ca3d 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java
@@ -170,8 +170,8 @@ public class TestPerFieldKnnVectorsFormat extends BaseKnnVectorsFormatTestCase {
try (Directory directory = newDirectory()) {
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
KnnVectorsFormat format1 =
- new KnnVectorsFormatMaxDims32(new Lucene99HnswVectorsFormat(16, 100, null));
- KnnVectorsFormat format2 = new Lucene99HnswVectorsFormat(16, 100, null);
+ new KnnVectorsFormatMaxDims32(new Lucene99HnswVectorsFormat(16, 100));
+ KnnVectorsFormat format2 = new Lucene99HnswVectorsFormat(16, 100);
iwc.setCodec(
new AssertingCodec() {
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
index 368f6f9be10..32f325970ab 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
@@ -113,8 +113,7 @@ public class TestKnnGraph extends LuceneTestCase {
return new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene99HnswVectorsFormat(
- M, HnswGraphBuilder.DEFAULT_BEAM_WIDTH, null);
+ return new Lucene99HnswVectorsFormat(M, HnswGraphBuilder.DEFAULT_BEAM_WIDTH);
}
};
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
index 015f45202d0..30105c2167b 100644
--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
@@ -175,7 +175,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {
return new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene99HnswVectorsFormat(M, beamWidth, null);
+ return new Lucene99HnswVectorsFormat(M, beamWidth);
}
};
}
@@ -247,7 +247,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {
return new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene99HnswVectorsFormat(M, beamWidth, null);
+ return new Lucene99HnswVectorsFormat(M, beamWidth);
}
};
}
@@ -308,7 +308,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {
return new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene99HnswVectorsFormat(M, beamWidth, null);
+ return new Lucene99HnswVectorsFormat(M, beamWidth);
}
};
}
@@ -322,7 +322,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {
return new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene99HnswVectorsFormat(M, beamWidth, null);
+ return new Lucene99HnswVectorsFormat(M, beamWidth);
}
};
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java
index 4cd473cc01f..11aa4aced97 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java
@@ -32,12 +32,12 @@ public class ConfigurableMCodec extends FilterCodec {
public ConfigurableMCodec() {
super("ConfigurableMCodec", TestUtil.getDefaultCodec());
- knnVectorsFormat = new Lucene99HnswVectorsFormat(128, 100, null);
+ knnVectorsFormat = new Lucene99HnswVectorsFormat(128, 100);
}
public ConfigurableMCodec(int maxConn) {
super("ConfigurableMCodec", TestUtil.getDefaultCodec());
- knnVectorsFormat = new Lucene99HnswVectorsFormat(maxConn, 100, null);
+ knnVectorsFormat = new Lucene99HnswVectorsFormat(maxConn, 100);
}
@Override