You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ti...@apache.org on 2026/06/15 16:26:49 UTC
(lucene) branch main updated: Reuse scratch int[] for ordinal translation (#16228)
This is an automated email from the ASF dual-hosted git repository.
Tim-Brooks pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new aef501aafd9 Reuse scratch int[] for ordinal translation (#16228)
aef501aafd9 is described below
commit aef501aafd9a0d80b6ddedba2baf06dec02aa94e
Author: Tim Brooks <ti...@timbrooks.org>
AuthorDate: Mon Jun 15 10:26:43 2026 -0600
Reuse scratch int[] for ordinal translation (#16228)
Adds a lazily-allocated 1024-element int[] (4 KB) to
SharedIndexingScratch. SortedDocValuesWriter and
SortedSetDocValuesWriter use it in place of per-call allocations for
ordinal-to-hash translation, falling back to ad-hoc allocation for
dictionaries > 1024.
---
lucene/CHANGES.txt | 3 +++
.../java/org/apache/lucene/index/IndexingChain.java | 6 ++++--
.../apache/lucene/index/SharedIndexingScratch.java | 18 ++++++++++++++++++
.../apache/lucene/index/SortedDocValuesWriter.java | 21 ++++++++++++++++-----
.../lucene/index/SortedSetDocValuesWriter.java | 13 ++++++++++---
5 files changed, 51 insertions(+), 10 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 38b7958904a..da64a61cc8c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -315,6 +315,7 @@ API Changes
* GITHUB#16224 Add TokenStreamColumn for experimental columnar batch indexing. (Tim Brooks)
+
New Features
---------------------
@@ -531,6 +532,8 @@ Optimizations
* GITHUB#16257: Support JDK 26 in Panama Vectorization Provider. (Chris Hegarty)
+* GITHUB#16228: Reuse scratch int[] for ordinal translation. (Tim Brooks)
+
Bug Fixes
---------------------
* GITHUB#15754: Fix HTMLStripCharFilter to prevent tags from incorrectly consuming subsequent
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index 9f99330fc2f..5e81ddb2b0a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -1347,13 +1347,15 @@ final class IndexingChain implements Accountable {
pf.docValuesWriter = new BinaryDocValuesWriter(fi, bytesUsed);
break;
case SORTED:
- pf.docValuesWriter = new SortedDocValuesWriter(fi, bytesUsed, docValuesBytePool);
+ pf.docValuesWriter =
+ new SortedDocValuesWriter(fi, bytesUsed, docValuesBytePool, sharedIndexingScratch);
break;
case SORTED_NUMERIC:
pf.docValuesWriter = new SortedNumericDocValuesWriter(fi, bytesUsed);
break;
case SORTED_SET:
- pf.docValuesWriter = new SortedSetDocValuesWriter(fi, bytesUsed, docValuesBytePool);
+ pf.docValuesWriter =
+ new SortedSetDocValuesWriter(fi, bytesUsed, docValuesBytePool, sharedIndexingScratch);
break;
default:
throw new AssertionError("unrecognized DocValues.Type: " + dvType);
diff --git a/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java b/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java
index 542a2acee0b..0dd86500a7f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java
@@ -32,8 +32,12 @@ final class SharedIndexingScratch {
/** Size in bytes of the shared byte scratch buffer. */
static final int BYTES_SCRATCH_SIZE = 4 * 1024;
+ /** Number of ints in the shared int scratch buffer (4 KB). */
+ static final int INTS_SCRATCH_SIZE = 1024;
+
private final Counter bytesUsed;
private byte[] bytesScratchBuffer;
+ private int[] intsScratchBuffer;
SharedIndexingScratch(Counter bytesUsed) {
this.bytesUsed = bytesUsed;
@@ -52,4 +56,18 @@ final class SharedIndexingScratch {
}
return bytesScratchBuffer;
}
+
+ /**
+ * Returns the shared int scratch buffer, allocating it on the first call and tracking its RAM via
+ * the {@link Counter} supplied at construction.
+ *
+ * <p>Callers must treat the returned array as transient scratch.
+ */
+ int[] intsScratch() {
+ if (intsScratchBuffer == null) {
+ intsScratchBuffer = new int[INTS_SCRATCH_SIZE];
+ bytesUsed.addAndGet((long) INTS_SCRATCH_SIZE * Integer.BYTES);
+ }
+ return intsScratchBuffer;
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
index 23b71abf1c3..714cc7a792f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
@@ -48,14 +48,17 @@ class SortedDocValuesWriter extends DocValuesWriter<SortedDocValues> {
private long bytesUsed; // this currently only tracks differences in 'pending'
private final FieldInfo fieldInfo;
private int lastDocID = -1;
+ private final SharedIndexingScratch scratch;
private PackedLongValues finalOrds;
private int[] finalSortedValues;
private int[] finalOrdMap;
- public SortedDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed, ByteBlockPool pool) {
+ public SortedDocValuesWriter(
+ FieldInfo fieldInfo, Counter iwBytesUsed, ByteBlockPool pool, SharedIndexingScratch scratch) {
this.fieldInfo = fieldInfo;
this.iwBytesUsed = iwBytesUsed;
+ this.scratch = scratch;
hash =
new BytesRefHash(
pool,
@@ -118,8 +121,12 @@ class SortedDocValuesWriter extends DocValuesWriter<SortedDocValues> {
* <p>All ordinals must be in {@code [0, dictionary.length)}.
*/
void addOrdinalTuples(int baseDocID, List<BytesRef> dictionary, OrdinalsTupleCursor cursor) {
- int[] ordToHash = new int[dictionary.size()];
- Arrays.fill(ordToHash, -1);
+ int dictSize = dictionary.size();
+ int[] ordToHash =
+ dictSize <= SharedIndexingScratch.INTS_SCRATCH_SIZE
+ ? scratch.intsScratch()
+ : new int[dictSize];
+ Arrays.fill(ordToHash, 0, dictSize, -1);
int batchDocID;
while ((batchDocID = cursor.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
int docID = baseDocID + batchDocID;
@@ -152,8 +159,12 @@ class SortedDocValuesWriter extends DocValuesWriter<SortedDocValues> {
return;
}
assert firstDocID > lastDocID;
- int[] ordToHash = new int[dictionary.size()];
- Arrays.fill(ordToHash, -1);
+ int dictSize = dictionary.size();
+ int[] ordToHash =
+ dictSize <= SharedIndexingScratch.INTS_SCRATCH_SIZE
+ ? scratch.intsScratch()
+ : new int[dictSize];
+ Arrays.fill(ordToHash, 0, dictSize, -1);
int processed = 0;
try {
while (processed < n) {
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
index 3c6f4a2d5ff..edaa40e5698 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
@@ -55,15 +55,18 @@ class SortedSetDocValuesWriter extends DocValuesWriter<SortedSetDocValues> {
private int[] currentValues = new int[8];
private int currentUpto;
private int maxCount;
+ private final SharedIndexingScratch scratch;
private PackedLongValues finalOrds;
private PackedLongValues finalOrdCounts;
private int[] finalSortedValues;
private int[] finalOrdMap;
- SortedSetDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed, ByteBlockPool pool) {
+ SortedSetDocValuesWriter(
+ FieldInfo fieldInfo, Counter iwBytesUsed, ByteBlockPool pool, SharedIndexingScratch scratch) {
this.fieldInfo = fieldInfo;
this.iwBytesUsed = iwBytesUsed;
+ this.scratch = scratch;
hash =
new BytesRefHash(
pool,
@@ -110,8 +113,12 @@ class SortedSetDocValuesWriter extends DocValuesWriter<SortedSetDocValues> {
* batch-local and are offset by {@code baseDocID} to produce segment-level ids.
*/
void addOrdinalTuples(int baseDocID, List<BytesRef> dictionary, OrdinalsTupleCursor cursor) {
- int[] ordToHash = new int[dictionary.size()];
- Arrays.fill(ordToHash, -1);
+ int dictSize = dictionary.size();
+ int[] ordToHash =
+ dictSize <= SharedIndexingScratch.INTS_SCRATCH_SIZE
+ ? scratch.intsScratch()
+ : new int[dictSize];
+ Arrays.fill(ordToHash, 0, dictSize, -1);
int batchDocID;
while ((batchDocID = cursor.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
int docID = baseDocID + batchDocID;