You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by pa...@apache.org on 2023/12/14 00:13:07 UTC

(lucene) branch branch_9x updated: Refactor around NeighborArray (#12910)

This is an automated email from the ASF dual-hosted git repository.

patrickz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_9x by this push:
     new ceec058767b Refactor around NeighborArray  (#12910)
ceec058767b is described below

commit ceec058767ba17da2cc6ccf9cb7a0bb3eadce381
Author: Patrick Zhai <zh...@users.noreply.github.com>
AuthorDate: Thu Dec 14 09:03:44 2023 +0900

    Refactor around NeighborArray  (#12910)
---
 lucene/CHANGES.txt                                 |   2 +
 .../lucene92/Lucene92HnswVectorsWriter.java        |   2 +-
 .../lucene94/Lucene94HnswVectorsWriter.java        |   4 +-
 .../lucene95/Lucene95HnswVectorsWriter.java        |   4 +-
 .../codecs/lucene99/Lucene99HnswVectorsWriter.java |   4 +-
 .../util/hnsw/HnswConcurrentMergeBuilder.java      |   2 +-
 .../apache/lucene/util/hnsw/HnswGraphBuilder.java  |  70 +---------
 .../apache/lucene/util/hnsw/HnswGraphSearcher.java |   2 +-
 .../org/apache/lucene/util/hnsw/NeighborArray.java | 152 ++++++++++++++++-----
 .../apache/lucene/util/hnsw/OnHeapHnswGraph.java   |   2 +-
 .../apache/lucene/util/hnsw/HnswGraphTestCase.java |   4 +-
 .../apache/lucene/util/hnsw/TestNeighborArray.java |   4 +-
 12 files changed, 139 insertions(+), 113 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a3a9f312ce7..d3ea38558f6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -31,6 +31,8 @@ Improvements
 
 * GITHUB#12812: Avoid overflows and false negatives in int slice buffer filled-with-zeros assertion. (Stefan Vodita)
 
+* GITHUB#12910: Refactor around NeighborArray to make it more self-contained. (Patrick Zhai)
+
 Optimizations
 ---------------------
 (No changes)
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.java
index ee50858ff13..f4bd21f72e3 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92HnswVectorsWriter.java
@@ -297,7 +297,7 @@ public final class Lucene92HnswVectorsWriter extends BufferingKnnVectorsWriter {
         int size = neighbors.size();
         vectorIndex.writeInt(size);
         // Destructively modify; it's ok we are discarding it after this
-        int[] nnodes = neighbors.node();
+        int[] nnodes = neighbors.nodes();
         Arrays.sort(nnodes, 0, size);
         for (int i = 0; i < size; i++) {
           int nnode = nnodes[i];
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java
index 604c6e97caf..6d3ffbb74e9 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java
@@ -369,7 +369,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
     vectorIndex.writeInt(size);
 
     // Destructively modify; it's ok we are discarding it after this
-    int[] nnodes = neighbors.node();
+    int[] nnodes = neighbors.nodes();
     for (int i = 0; i < size; i++) {
       nnodes[i] = oldToNewMap[nnodes[i]];
     }
@@ -506,7 +506,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
         int size = neighbors.size();
         vectorIndex.writeInt(size);
         // Destructively modify; it's ok we are discarding it after this
-        int[] nnodes = neighbors.node();
+        int[] nnodes = neighbors.nodes();
         Arrays.sort(nnodes, 0, size);
         for (int i = 0; i < size; i++) {
           int nnode = nnodes[i];
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java
index c62ddd4323f..7b931c79832 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95HnswVectorsWriter.java
@@ -396,7 +396,7 @@ public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter {
     vectorIndex.writeVInt(size);
 
     // Destructively modify; it's ok we are discarding it after this
-    int[] nnodes = neighbors.node();
+    int[] nnodes = neighbors.nodes();
     for (int i = 0; i < size; i++) {
       nnodes[i] = oldToNewMap[nnodes[i]];
     }
@@ -556,7 +556,7 @@ public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter {
         long offsetStart = vectorIndex.getFilePointer();
         vectorIndex.writeVInt(size);
         // Destructively modify; it's ok we are discarding it after this
-        int[] nnodes = neighbors.node();
+        int[] nnodes = neighbors.nodes();
         Arrays.sort(nnodes, 0, size);
         // Now that we have sorted, do delta encoding to minimize the required bits to store the
         // information
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
index f39069ba981..4e1814cfd23 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
@@ -319,7 +319,7 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {
     vectorIndex.writeVInt(size);
 
     // Destructively modify; it's ok we are discarding it after this
-    int[] nnodes = neighbors.node();
+    int[] nnodes = neighbors.nodes();
     for (int i = 0; i < size; i++) {
       nnodes[i] = oldToNewMap[nnodes[i]];
     }
@@ -415,7 +415,7 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {
         long offsetStart = vectorIndex.getFilePointer();
         vectorIndex.writeVInt(size);
         // Destructively modify; it's ok we are discarding it after this
-        int[] nnodes = neighbors.node();
+        int[] nnodes = neighbors.nodes();
         Arrays.sort(nnodes, 0, size);
         // Now that we have sorted, do delta encoding to minimize the required bits to store the
         // information
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswConcurrentMergeBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswConcurrentMergeBuilder.java
index 07d1ae2f698..db48960df86 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswConcurrentMergeBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswConcurrentMergeBuilder.java
@@ -201,7 +201,7 @@ public class HnswConcurrentMergeBuilder implements HnswBuilder {
           nodeBuffer = new int[neighborArray.size()];
         }
         size = neighborArray.size();
-        if (size >= 0) System.arraycopy(neighborArray.node, 0, nodeBuffer, 0, size);
+        if (size >= 0) System.arraycopy(neighborArray.nodes(), 0, nodeBuffer, 0, size);
       } finally {
         neighborArray.rwlock.readLock().unlock();
       }
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
index a1781067fcc..b794244105e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
@@ -310,15 +310,11 @@ public class HnswGraphBuilder implements HnswBuilder {
       if (mask[i] == false) {
         continue;
       }
-      int nbr = candidates.node[i];
+      int nbr = candidates.nodes()[i];
       NeighborArray nbrsOfNbr = hnsw.getNeighbors(level, nbr);
       nbrsOfNbr.rwlock.writeLock().lock();
       try {
-        nbrsOfNbr.addOutOfOrder(node, candidates.score[i]);
-        if (nbrsOfNbr.size() > maxConnOnLevel) {
-          int indexToRemove = findWorstNonDiverse(nbrsOfNbr, nbr);
-          nbrsOfNbr.removeIndex(indexToRemove);
-        }
+        nbrsOfNbr.addAndEnsureDiversity(node, candidates.scores()[i], nbr, scorerSupplier);
       } finally {
         nbrsOfNbr.rwlock.writeLock().unlock();
       }
@@ -336,8 +332,8 @@ public class HnswGraphBuilder implements HnswBuilder {
     for (int i = candidates.size() - 1; neighbors.size() < maxConnOnLevel && i >= 0; i--) {
       // compare each neighbor (in distance order) against the closer neighbors selected so far,
       // only adding it if it is closer to the target than to any of the other selected neighbors
-      int cNode = candidates.node[i];
-      float cScore = candidates.score[i];
+      int cNode = candidates.nodes()[i];
+      float cScore = candidates.scores()[i];
       assert cNode <= hnsw.maxNodeId();
       if (diversityCheck(cNode, cScore, neighbors)) {
         mask[i] = true;
@@ -371,7 +367,7 @@ public class HnswGraphBuilder implements HnswBuilder {
       throws IOException {
     RandomVectorScorer scorer = scorerSupplier.scorer(candidate);
     for (int i = 0; i < neighbors.size(); i++) {
-      float neighborSimilarity = scorer.score(neighbors.node[i]);
+      float neighborSimilarity = scorer.score(neighbors.nodes()[i]);
       if (neighborSimilarity >= score) {
         return false;
       }
@@ -379,62 +375,6 @@ public class HnswGraphBuilder implements HnswBuilder {
     return true;
   }
 
-  /**
-   * Find first non-diverse neighbour among the list of neighbors starting from the most distant
-   * neighbours
-   */
-  private int findWorstNonDiverse(NeighborArray neighbors, int nodeOrd) throws IOException {
-    RandomVectorScorer scorer = scorerSupplier.scorer(nodeOrd);
-    int[] uncheckedIndexes = neighbors.sort(scorer);
-    if (uncheckedIndexes == null) {
-      // all nodes are checked, we will directly return the most distant one
-      return neighbors.size() - 1;
-    }
-    int uncheckedCursor = uncheckedIndexes.length - 1;
-    for (int i = neighbors.size() - 1; i > 0; i--) {
-      if (uncheckedCursor < 0) {
-        // no unchecked node left
-        break;
-      }
-      if (isWorstNonDiverse(i, neighbors, uncheckedIndexes, uncheckedCursor)) {
-        return i;
-      }
-      if (i == uncheckedIndexes[uncheckedCursor]) {
-        uncheckedCursor--;
-      }
-    }
-    return neighbors.size() - 1;
-  }
-
-  private boolean isWorstNonDiverse(
-      int candidateIndex, NeighborArray neighbors, int[] uncheckedIndexes, int uncheckedCursor)
-      throws IOException {
-    float minAcceptedSimilarity = neighbors.score[candidateIndex];
-    RandomVectorScorer scorer = scorerSupplier.scorer(neighbors.node[candidateIndex]);
-    if (candidateIndex == uncheckedIndexes[uncheckedCursor]) {
-      // the candidate itself is unchecked
-      for (int i = candidateIndex - 1; i >= 0; i--) {
-        float neighborSimilarity = scorer.score(neighbors.node[i]);
-        // candidate node is too similar to node i given its score relative to the base node
-        if (neighborSimilarity >= minAcceptedSimilarity) {
-          return true;
-        }
-      }
-    } else {
-      // else we just need to make sure candidate does not violate diversity with the (newly
-      // inserted) unchecked nodes
-      assert candidateIndex > uncheckedIndexes[uncheckedCursor];
-      for (int i = uncheckedCursor; i >= 0; i--) {
-        float neighborSimilarity = scorer.score(neighbors.node[uncheckedIndexes[i]]);
-        // candidate node is too similar to node i given its score relative to the base node
-        if (neighborSimilarity >= minAcceptedSimilarity) {
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
   private static int getRandomGraphLevel(double ml, SplittableRandom random) {
     double randDouble;
     do {
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java
index 2aa5389ff58..46d6c93d52c 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java
@@ -308,7 +308,7 @@ public class HnswGraphSearcher {
     @Override
     int graphNextNeighbor(HnswGraph graph) {
       if (++upto < cur.size()) {
-        return cur.node[upto];
+        return cur.nodes()[upto];
       }
       return NO_MORE_DOCS;
     }
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/NeighborArray.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/NeighborArray.java
index 086a5ada305..7d1ed069c29 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/NeighborArray.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/NeighborArray.java
@@ -34,14 +34,14 @@ import org.apache.lucene.util.ArrayUtil;
 public class NeighborArray {
   private final boolean scoresDescOrder;
   private int size;
-  float[] score;
-  int[] node;
+  private final float[] scores;
+  private final int[] nodes;
   private int sortedNodeSize;
   public final ReadWriteLock rwlock = new ReentrantReadWriteLock(true);
 
   public NeighborArray(int maxSize, boolean descOrder) {
-    node = new int[maxSize];
-    score = new float[maxSize];
+    nodes = new int[maxSize];
+    scores = new float[maxSize];
     this.scoresDescOrder = descOrder;
   }
 
@@ -51,37 +51,56 @@ public class NeighborArray {
    */
   public void addInOrder(int newNode, float newScore) {
     assert size == sortedNodeSize : "cannot call addInOrder after addOutOfOrder";
-    if (size == node.length) {
-      node = ArrayUtil.grow(node);
-      score = ArrayUtil.growExact(score, node.length);
+    if (size == nodes.length) {
+      throw new IllegalStateException("No growth is allowed");
     }
     if (size > 0) {
-      float previousScore = score[size - 1];
+      float previousScore = scores[size - 1];
       assert ((scoresDescOrder && (previousScore >= newScore))
               || (scoresDescOrder == false && (previousScore <= newScore)))
           : "Nodes are added in the incorrect order! Comparing "
               + newScore
               + " to "
-              + Arrays.toString(ArrayUtil.copyOfSubArray(score, 0, size));
+              + Arrays.toString(ArrayUtil.copyOfSubArray(scores, 0, size));
     }
-    node[size] = newNode;
-    score[size] = newScore;
+    nodes[size] = newNode;
+    scores[size] = newScore;
     ++size;
     ++sortedNodeSize;
   }
 
   /** Add node and newScore but do not insert as sorted */
   public void addOutOfOrder(int newNode, float newScore) {
-    if (size == node.length) {
-      node = ArrayUtil.grow(node);
-      score = ArrayUtil.growExact(score, node.length);
+    if (size == nodes.length) {
+      throw new IllegalStateException("No growth is allowed");
     }
 
-    score[size] = newScore;
-    node[size] = newNode;
+    scores[size] = newScore;
+    nodes[size] = newNode;
     size++;
   }
 
+  /**
+   * In addition to {@link #addOutOfOrder(int, float)}, this function will also remove the
+   * least-diverse node if the node array is full after insertion
+   *
+   * <p>In multi-threading environment, this method need to be locked as it will be called by
+   * multiple threads while other add method is only supposed to be called by one thread.
+   *
+   * @param nodeId node Id of the owner of this NeighbourArray
+   */
+  public void addAndEnsureDiversity(
+      int newNode, float newScore, int nodeId, RandomVectorScorerSupplier scorerSupplier)
+      throws IOException {
+    addOutOfOrder(newNode, newScore);
+    if (size < nodes.length) {
+      return;
+    }
+    // we're oversize, need to do diversity check and pop out the least diverse neighbour
+    removeIndex(findWorstNonDiverse(nodeId, scorerSupplier));
+    assert size == nodes.length - 1;
+  }
+
   /**
    * Sort the array according to scores, and return the sorted indexes of previous unsorted nodes
    * (unchecked nodes)
@@ -89,7 +108,7 @@ public class NeighborArray {
    * @return indexes of newly sorted (unchecked) nodes, in ascending order, or null if the array is
    *     already fully sorted
    */
-  public int[] sort(RandomVectorScorer scorer) throws IOException {
+  int[] sort(RandomVectorScorer scorer) throws IOException {
     if (size == sortedNodeSize) {
       // all nodes checked and sorted
       return null;
@@ -98,6 +117,9 @@ public class NeighborArray {
     int[] uncheckedIndexes = new int[size - sortedNodeSize];
     int count = 0;
     while (sortedNodeSize != size) {
+      // TODO: Instead of do an array copy on every insertion, I think we can do better here:
+      //       Remember the insertion point of each unsorted node and insert them altogether
+      //       We can save several array copy by doing that
       uncheckedIndexes[count] = insertSortedInternal(scorer); // sortedNodeSize is increased inside
       for (int i = 0; i < count; i++) {
         if (uncheckedIndexes[i] >= uncheckedIndexes[count]) {
@@ -114,8 +136,8 @@ public class NeighborArray {
   /** insert the first unsorted node into its sorted position */
   private int insertSortedInternal(RandomVectorScorer scorer) throws IOException {
     assert sortedNodeSize < size : "Call this method only when there's unsorted node";
-    int tmpNode = node[sortedNodeSize];
-    float tmpScore = score[sortedNodeSize];
+    int tmpNode = nodes[sortedNodeSize];
+    float tmpScore = scores[sortedNodeSize];
 
     if (Float.isNaN(tmpScore)) {
       tmpScore = scorer.score(tmpNode);
@@ -126,11 +148,11 @@ public class NeighborArray {
             ? descSortFindRightMostInsertionPoint(tmpScore, sortedNodeSize)
             : ascSortFindRightMostInsertionPoint(tmpScore, sortedNodeSize);
     System.arraycopy(
-        node, insertionPoint, node, insertionPoint + 1, sortedNodeSize - insertionPoint);
+        nodes, insertionPoint, nodes, insertionPoint + 1, sortedNodeSize - insertionPoint);
     System.arraycopy(
-        score, insertionPoint, score, insertionPoint + 1, sortedNodeSize - insertionPoint);
-    node[insertionPoint] = tmpNode;
-    score[insertionPoint] = tmpScore;
+        scores, insertionPoint, scores, insertionPoint + 1, sortedNodeSize - insertionPoint);
+    nodes[insertionPoint] = tmpNode;
+    scores[insertionPoint] = tmpScore;
     ++sortedNodeSize;
     return insertionPoint;
   }
@@ -150,12 +172,12 @@ public class NeighborArray {
    *
    * @lucene.internal
    */
-  public int[] node() {
-    return node;
+  public int[] nodes() {
+    return nodes;
   }
 
-  public float[] score() {
-    return score;
+  public float[] scores() {
+    return scores;
   }
 
   public void clear() {
@@ -163,14 +185,18 @@ public class NeighborArray {
     sortedNodeSize = 0;
   }
 
-  public void removeLast() {
+  void removeLast() {
     size--;
     sortedNodeSize = Math.min(sortedNodeSize, size);
   }
 
-  public void removeIndex(int idx) {
-    System.arraycopy(node, idx + 1, node, idx, size - idx - 1);
-    System.arraycopy(score, idx + 1, score, idx, size - idx - 1);
+  void removeIndex(int idx) {
+    if (idx == size - 1) {
+      removeLast();
+      return;
+    }
+    System.arraycopy(nodes, idx + 1, nodes, idx, size - idx - 1);
+    System.arraycopy(scores, idx + 1, scores, idx, size - idx - 1);
     if (idx < sortedNodeSize) {
       sortedNodeSize--;
     }
@@ -183,10 +209,11 @@ public class NeighborArray {
   }
 
   private int ascSortFindRightMostInsertionPoint(float newScore, int bound) {
-    int insertionPoint = Arrays.binarySearch(score, 0, bound, newScore);
+    int insertionPoint = Arrays.binarySearch(scores, 0, bound, newScore);
     if (insertionPoint >= 0) {
       // find the right most position with the same score
-      while ((insertionPoint < bound - 1) && (score[insertionPoint + 1] == score[insertionPoint])) {
+      while ((insertionPoint < bound - 1)
+          && (scores[insertionPoint + 1] == scores[insertionPoint])) {
         insertionPoint++;
       }
       insertionPoint++;
@@ -201,9 +228,66 @@ public class NeighborArray {
     int end = bound - 1;
     while (start <= end) {
       int mid = (start + end) / 2;
-      if (score[mid] < newScore) end = mid - 1;
+      if (scores[mid] < newScore) end = mid - 1;
       else start = mid + 1;
     }
     return start;
   }
+
+  /**
+   * Find first non-diverse neighbour among the list of neighbors starting from the most distant
+   * neighbours
+   */
+  private int findWorstNonDiverse(int nodeOrd, RandomVectorScorerSupplier scorerSupplier)
+      throws IOException {
+    RandomVectorScorer scorer = scorerSupplier.scorer(nodeOrd);
+    int[] uncheckedIndexes = sort(scorer);
+    assert uncheckedIndexes != null : "We will always have something unchecked";
+    int uncheckedCursor = uncheckedIndexes.length - 1;
+    for (int i = size - 1; i > 0; i--) {
+      if (uncheckedCursor < 0) {
+        // no unchecked node left
+        break;
+      }
+      if (isWorstNonDiverse(i, uncheckedIndexes, uncheckedCursor, scorerSupplier)) {
+        return i;
+      }
+      if (i == uncheckedIndexes[uncheckedCursor]) {
+        uncheckedCursor--;
+      }
+    }
+    return size - 1;
+  }
+
+  private boolean isWorstNonDiverse(
+      int candidateIndex,
+      int[] uncheckedIndexes,
+      int uncheckedCursor,
+      RandomVectorScorerSupplier scorerSupplier)
+      throws IOException {
+    float minAcceptedSimilarity = scores[candidateIndex];
+    RandomVectorScorer scorer = scorerSupplier.scorer(nodes[candidateIndex]);
+    if (candidateIndex == uncheckedIndexes[uncheckedCursor]) {
+      // the candidate itself is unchecked
+      for (int i = candidateIndex - 1; i >= 0; i--) {
+        float neighborSimilarity = scorer.score(nodes[i]);
+        // candidate node is too similar to node i given its score relative to the base node
+        if (neighborSimilarity >= minAcceptedSimilarity) {
+          return true;
+        }
+      }
+    } else {
+      // else we just need to make sure candidate does not violate diversity with the (newly
+      // inserted) unchecked nodes
+      assert candidateIndex > uncheckedIndexes[uncheckedCursor];
+      for (int i = uncheckedCursor; i >= 0; i--) {
+        float neighborSimilarity = scorer.score(nodes[uncheckedIndexes[i]]);
+        // candidate node is too similar to node i given its score relative to the base node
+        if (neighborSimilarity >= minAcceptedSimilarity) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
index 60c3c3ad762..08c67a2af6d 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
@@ -164,7 +164,7 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
   @Override
   public int nextNeighbor() {
     if (++upto < cur.size()) {
-      return cur.node[upto];
+      return cur.nodes()[upto];
     }
     return NO_MORE_DOCS;
   }
diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
index 8d75d3af50e..b4c68c90d16 100644
--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java
@@ -455,7 +455,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {
 
     for (int i = 0; i < nDoc; i++) {
       NeighborArray neighbors = hnsw.getNeighbors(0, i);
-      int[] nnodes = neighbors.node;
+      int[] nnodes = neighbors.nodes();
       for (int j = 0; j < neighbors.size(); j++) {
         // all neighbors should be valid node ids.
         assertTrue(nnodes[j] < nDoc);
@@ -852,7 +852,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {
   private void assertLevel0Neighbors(OnHeapHnswGraph graph, int node, int... expected) {
     Arrays.sort(expected);
     NeighborArray nn = graph.getNeighbors(0, node);
-    int[] actual = ArrayUtil.copyOfSubArray(nn.node, 0, nn.size());
+    int[] actual = ArrayUtil.copyOfSubArray(nn.nodes(), 0, nn.size());
     Arrays.sort(actual);
     assertArrayEquals(
         "expected: " + Arrays.toString(expected) + " actual: " + Arrays.toString(actual),
diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/TestNeighborArray.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/TestNeighborArray.java
index 17b709d2e66..d23f7dd61fe 100644
--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/TestNeighborArray.java
+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/TestNeighborArray.java
@@ -215,13 +215,13 @@ public class TestNeighborArray extends LuceneTestCase {
 
   private void assertScoresEqual(float[] scores, NeighborArray neighbors) {
     for (int i = 0; i < scores.length; i++) {
-      assertEquals(scores[i], neighbors.score[i], 0.01f);
+      assertEquals(scores[i], neighbors.scores()[i], 0.01f);
     }
   }
 
   private void assertNodesEqual(int[] nodes, NeighborArray neighbors) {
     for (int i = 0; i < nodes.length; i++) {
-      assertEquals(nodes[i], neighbors.node[i]);
+      assertEquals(nodes[i], neighbors.nodes()[i]);
     }
   }