You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2026/02/02 16:44:09 UTC
(lucene) branch branch_10x updated: Bulk score hnsw diversity check (#15607)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 39969e5cbfe Bulk score hnsw diversity check (#15607)
39969e5cbfe is described below
commit 39969e5cbfe50240d7d48826936539eccee79993
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Mon Feb 2 11:00:10 2026 -0500
Bulk score hnsw diversity check (#15607)
* Utilize bulk scoring interface during HNSW graph builder diversity check
* iter
* iter
* iter
* adding changes, adjusting bulk chunk size
* iter
* fixing chunk size
* no tail needed, :facepalm:
* iter
* iter
---
lucene/CHANGES.txt | 3 +++
.../org/apache/lucene/util/hnsw/HnswGraphBuilder.java | 16 +++++++++++++---
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 78599983ff7..fa7f9acb523 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -174,6 +174,9 @@ Optimizations
* GITHUB#15632: Use a coarser-grained competitive iterator with lower construction costs for
numeric sorts against fields with DocValuesSkippers. (Alan Woodward)
+* GITHUB#15607: Utilize bulk scoring for diversity checking when building HNSW vector indices. This results
+ in some performance improvements during indexing and segment merges. (Ben Trent)
+
Bug Fixes
---------------------
* GITHUB#14161: PointInSetQuery's constructor now throws IllegalArgumentException
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
index d23885c8648..bd38843dc68 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
@@ -64,9 +64,13 @@ public class HnswGraphBuilder implements HnswBuilder {
@SuppressWarnings("NonFinalStaticField")
public static long randSeed = DEFAULT_RAND_SEED;
+ private static final int MAX_BULK_SCORE_NODES = 8;
+
protected final int M; // max number of connections on upper layers
private final double ml;
+ private final int[] bulkScoreNodes; // for bulk scoring
+ private final float[] bulkScores; // for bulk scoring
private final SplittableRandom random;
protected final UpdateableRandomVectorScorer scorer;
protected final HnswGraphSearcher graphSearcher;
@@ -177,6 +181,10 @@ public class HnswGraphBuilder implements HnswBuilder {
this.hnsw = hnsw;
this.hnswLock = hnswLock;
this.graphSearcher = graphSearcher;
+ // pick a number that keeps us from scoring TOO much for diversity checking
+ // but enough to take advantage of bulk scoring
+ this.bulkScoreNodes = new int[MAX_BULK_SCORE_NODES];
+ this.bulkScores = new float[MAX_BULK_SCORE_NODES];
entryCandidates = new GraphBuilderKnnCollector(1);
beamCandidates = new GraphBuilderKnnCollector(beamWidth);
beamCandidates0 = new GraphBuilderKnnCollector(Math.min(beamWidth / 2, M * 3));
@@ -491,9 +499,11 @@ public class HnswGraphBuilder implements HnswBuilder {
*/
private boolean diversityCheck(float score, NeighborArray neighbors, RandomVectorScorer scorer)
throws IOException {
- for (int i = 0; i < neighbors.size(); i++) {
- float neighborSimilarity = scorer.score(neighbors.nodes()[i]);
- if (neighborSimilarity >= score) {
+ final int bulkScoreChunk = Math.min((neighbors.size() + 1) / 2, bulkScoreNodes.length);
+ for (int scored = 0; scored < neighbors.size(); scored += bulkScoreChunk) {
+ int chunkSize = Math.min(bulkScoreChunk, neighbors.size() - scored);
+ System.arraycopy(neighbors.nodes(), scored, bulkScoreNodes, 0, chunkSize);
+ if (scorer.bulkScore(bulkScoreNodes, bulkScores, chunkSize) >= score) {
return false;
}
}