You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2026/03/23 20:17:06 UTC
(lucene) branch main updated: Improve BytesRefHash.add performance by optimize rehash operation (#15779)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 48379f06383 Improve BytesRefHash.add performance by optimize rehash operation (#15779)
48379f06383 is described below
commit 48379f06383329c1105dbd42e1b59ba608f0a1e3
Author: tyronecai <ty...@163.com>
AuthorDate: Tue Mar 24 04:17:00 2026 +0800
Improve BytesRefHash.add performance by optimize rehash operation (#15779)
* Optimize rehash in BytesRefHash
Precompute all term hash values sequentially at once
which is much faster than out-of-order computation
* Update changes.
* Tidy.
* reduce memory usage
---------
Co-authored-by: Dawid Weiss <da...@carrotsearch.com>
---
lucene/CHANGES.txt | 2 +
.../java/org/apache/lucene/util/BytesRefHash.java | 55 ++++++++++------------
2 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b69e8ddafc6..ea86c442197 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -256,6 +256,8 @@ Optimizations
* GITHUB#15024: Improve prefix sum computation in Lucene99HnswVectorsReader for faster neighbor decoding. (Luis Negrin)
+* GITHUB#15779: Improve BytesRefHash.add performance by optimize rehash operation (tyronecai)
+
Bug Fixes
---------------------
* GITHUB#15754: Fix HTMLStripCharFilter to prevent tags from incorrectly consuming subsequent
diff --git a/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java b/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java
index 2e1850344dc..f71c59f768e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java
+++ b/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java
@@ -433,40 +433,37 @@ public final class BytesRefHash implements Accountable {
private void rehash(final int newSize, boolean hashOnData) {
final int newMask = newSize - 1;
final int newHighMask = ~newMask;
- bytesUsed.addAndGet(Integer.BYTES * (long) newSize);
- final int[] newHash = new int[newSize];
- Arrays.fill(newHash, -1);
- for (int i = 0; i < hashSize; i++) {
- int e0 = ids[i];
- if (e0 != -1) {
- e0 &= hashMask;
- final int hashcode;
- int code;
- if (hashOnData) {
- hashcode = code = pool.hash(bytesStart[e0]);
- } else {
- code = bytesStart[e0];
- hashcode = 0;
- }
-
- int hashPos = code & newMask;
- assert hashPos >= 0;
-
- // Conflict; use linear probe to find an open slot
- // (see LUCENE-5604):
- while (newHash[hashPos] != -1) {
- code++;
- hashPos = code & newMask;
- }
-
- newHash[hashPos] = e0 | (hashcode & newHighMask);
+ bytesUsed.addAndGet(Integer.BYTES * (long) (newSize - ids.length));
+
+ ids = new int[newSize];
+ Arrays.fill(ids, -1);
+
+ // rebuild ids from terms in pool pointed by bytesStart
+ for (int id = 0; id < count; id++) {
+ final int hashcode;
+ int code;
+ if (hashOnData) {
+ hashcode = code = pool.hash(bytesStart[id]);
+ } else {
+ code = bytesStart[id];
+ hashcode = 0;
}
+
+ int hashPos = code & newMask;
+ assert hashPos >= 0;
+
+ // Conflict; use linear probe to find an open slot
+ // (see LUCENE-5604):
+ while (ids[hashPos] != -1) {
+ code++;
+ hashPos = code & newMask;
+ }
+
+ ids[hashPos] = id | (hashcode & newHighMask);
}
hashMask = newMask;
highMask = newHighMask;
- bytesUsed.addAndGet(Integer.BYTES * (long) -ids.length);
- ids = newHash;
hashSize = newSize;
hashHalfSize = newSize / 2;
}