You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2026/02/26 16:26:06 UTC
(lucene) branch main updated: Fix race and improve performance of HNSW graph building (#15590)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 3b8a0cdbcc6 Fix race and improve performance of HNSW graph building (#15590)
3b8a0cdbcc6 is described below
commit 3b8a0cdbcc6edacfd3ce1305f74c6540183d1ea8
Author: Viliam Durina <vi...@users.noreply.github.com>
AuthorDate: Thu Feb 26 17:25:56 2026 +0100
Fix race and improve performance of HNSW graph building (#15590)
* Fix race in OnHeapHnswGraph memory accounting
Previously, a volatile field `graphRamBytesUsed` was used. However, a non-atomic read-modify-write operation was used, which could lead to lost update in case multiple threads are updating the field.
The field was replaced with a `LongAdderr`.
See also the discussion in https://lists.apache.org/thread/xj8j0hx7nggo25471mybky1h9m4rrm85
* Update CHANGES
* Update CHANGES.txt
* Update CHANGES.txt
* Apply suggestion from @benwtrent
* formatting
---------
Co-authored-by: Benjamin Trent <be...@gmail.com>
Co-authored-by: Benjamin Trent <43...@users.noreply.github.com>
---
lucene/CHANGES.txt | 2 ++
.../apache/lucene/util/hnsw/OnHeapHnswGraph.java | 37 +++++++---------------
2 files changed, 14 insertions(+), 25 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4cf189855e0..272dd0210c6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -238,6 +238,8 @@ Bug Fixes
* GITHUB#15668: Fix UnsupportedOperationException in WeightedSpanTermExtractor by
implementing getFieldInfos in DelegatingLeafReader. (Simranjeet Singh)
+* GITHUB#15590: Fix race and improve performance of HNSW graph building (Viliam Durina)
+
* GITHUB#15702: Fix Potential NullPointException in BytesRefHash initialization#15702 (tyronecai)
* GITHUB#15731: Fix tessellator failure when a hole is touching another hole in the middle of a segment. In this case
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
index f2ceda091be..c8d73723746 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
@@ -21,6 +21,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.ArrayUtil;
@@ -63,7 +64,7 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
private int upto;
private NeighborArray cur;
- private volatile long graphRamBytesUsed;
+ private final LongAdder graphRamBytesUsed = new LongAdder();
/**
* ctor
@@ -83,7 +84,7 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
numNodes = INIT_SIZE;
}
this.graph = new NeighborArray[numNodes][];
- this.graphRamBytesUsed = RAM_BYTES_USED + RamUsageEstimator.shallowSizeOf(graph);
+ this.graphRamBytesUsed.add(RAM_BYTES_USED + RamUsageEstimator.shallowSizeOf(graph));
}
/**
@@ -160,27 +161,14 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
graph[node] = ArrayUtil.growExact(graph[node], level + 1);
}
- if (level == 0) {
- graph[node][level] =
- new NeighborArray(
- nsize0,
- true,
- l -> {
- assert l > 0;
- long bytesUsed = graphRamBytesUsed;
- graphRamBytesUsed = bytesUsed + l;
- });
- } else {
- graph[node][level] =
- new NeighborArray(
- nsize,
- true,
- l -> {
- assert l > 0;
- long bytesUsed = graphRamBytesUsed;
- graphRamBytesUsed = bytesUsed + l;
- });
- }
+ graph[node][level] =
+ new NeighborArray(
+ level == 0 ? nsize0 : nsize,
+ true,
+ l -> {
+ assert l > 0;
+ graphRamBytesUsed.add(l);
+ });
maxNodeId.accumulateAndGet(node, Math::max);
}
@@ -289,7 +277,6 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
}
}
- @SuppressWarnings({"unchecked", "rawtypes"})
private void generateLevelToNodes() {
if (lastFreezeSize == size()) {
return;
@@ -323,7 +310,7 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
*/
@Override
public long ramBytesUsed() {
- return graphRamBytesUsed;
+ return graphRamBytesUsed.sum();
}
@Override