You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by be...@apache.org on 2026/05/27 15:20:50 UTC
(lucene) branch main updated: Add bulk decode for dense NumericDocValues via longValues() (#16129)
This is an automated email from the ASF dual-hosted git repository.
benwtrent pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 96691018f58 Add bulk decode for dense NumericDocValues via longValues() (#16129)
96691018f58 is described below
commit 96691018f58b3d9a870e724842af3254a47f5c67
Author: Costin Leau <co...@users.noreply.github.com>
AuthorDate: Wed May 27 08:20:42 2026 -0700
Add bulk decode for dense NumericDocValues via longValues() (#16129)
* Optimize bulk numeric doc values
Add byte-aligned bulk decoding for dense numeric doc values so contiguous
batches can avoid per-document packed-value lookups.
* Add offset-aware longValues() overload
Add docsOffset and valuesOffset parameters following the
same convention as rangeIntoBitSet and System.arraycopy.
The 4-arg longValues delegates to the 6-arg variant.
Update bulkDecodeByteAlignedValues and all codec overrides
to thread offsets through, with loop variables starting at
the offset to avoid per-iteration addition.
Also expand the JMH benchmark to cover all byte-aligned
BPV values (24, 40, 48, 56) alongside power-of-two widths.
* Update CHANGES.txt
---------
Co-authored-by: Benjamin Trent <be...@gmail.com>
---
lucene/CHANGES.txt | 2 +
.../jmh/NumericDocValuesBulkDecodeBenchmark.java | 159 +++++++++++++++++++
.../codecs/lucene90/Lucene90DocValuesProducer.java | 175 +++++++++++++++++++++
.../org/apache/lucene/index/NumericDocValues.java | 25 ++-
.../DefaultDocValuesBulkDecodeSupport.java | 120 ++++++++++++++
.../DefaultVectorizationProvider.java | 5 +
.../vectorization/DocValuesBulkDecodeSupport.java | 45 ++++++
.../vectorization/VectorizationProvider.java | 7 +
.../vectorization/NativeVectorizationProvider.java | 5 +
.../PanamaDocValuesBulkDecodeSupport.java | 56 +++++++
.../vectorization/PanamaVectorizationProvider.java | 5 +
.../lucene90/TestLucene90DocValuesFormat.java | 103 ++++++++++++
12 files changed, 703 insertions(+), 4 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index aa0935250e6..7dc4cc7b7b4 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -434,6 +434,8 @@ Optimizations
* GITHUB#16107: Add method SparseFixedBitSet#set(int startIndex, int endIndex). (Ignacio Vera)
+* GITHUB#16129: Add bulk decode for dense NumericDocValues via longValues(). (Costin Leau)
+
* GITHUB#16126: Implement #intoBitSet and #docIDRunEnd in DocValuesWriter implementations. (Ignacio Vera)
Bug Fixes
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/NumericDocValuesBulkDecodeBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/NumericDocValuesBulkDecodeBenchmark.java
new file mode 100644
index 00000000000..66509e21993
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/NumericDocValuesBulkDecodeBenchmark.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * Benchmarks bulk retrieval of dense numeric doc values via {@link NumericDocValues#longValues}.
+ */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 3, time = 2)
+@Measurement(iterations = 5, time = 2)
+public class NumericDocValuesBulkDecodeBenchmark {
+
+ private Directory dir;
+ private DirectoryReader reader;
+ private NumericDocValues values;
+ private Path path;
+ private int[] docs;
+ private long[] valueBuffer;
+ private int nextStart;
+
+ @Param({"1000000"})
+ public int docCount;
+
+ @Param({"8", "16", "24", "32", "40", "48", "56", "64"})
+ public int bitsPerValue;
+
+ @Param({"128", "1024"})
+ public int batchSize;
+
+ @Param({"contiguous", "strided"})
+ public String accessPattern;
+
+ @Setup(Level.Trial)
+ public void setup() throws Exception {
+ path = Files.createTempDirectory("numericDocValuesBulkDecode");
+ dir = MMapDirectory.open(path);
+
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig());
+ Random random = new Random(0);
+ long mask = bitsPerValue == Long.SIZE ? -1L : (1L << bitsPerValue) - 1;
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ long value = bitsPerValue == Long.SIZE ? random.nextLong() : random.nextLong() & mask;
+ doc.add(new NumericDocValuesField("field", value));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ reader = DirectoryReader.open(writer);
+ writer.close();
+
+ values = reader.leaves().get(0).reader().getNumericDocValues("field");
+ docs = new int[batchSize];
+ valueBuffer = new long[batchSize];
+ }
+
+ @TearDown(Level.Trial)
+ public void tearDown() throws Exception {
+ reader.close();
+ dir.close();
+ if (Files.exists(path)) {
+ try (Stream<Path> walk = Files.walk(path)) {
+ walk.sorted(Comparator.reverseOrder())
+ .forEach(
+ p -> {
+ try {
+ Files.delete(p);
+ } catch (IOException _) {
+ }
+ });
+ }
+ }
+ }
+
+ @Benchmark
+ @Fork(
+ value = 1,
+ jvmArgsAppend = {"-Xmx2g", "-Xms2g", "-XX:+AlwaysPreTouch"})
+ public long longValuesDefaultProvider() throws IOException {
+ return readBatch();
+ }
+
+ @Benchmark
+ @Fork(
+ value = 1,
+ jvmArgsAppend = {
+ "--add-modules",
+ "jdk.incubator.vector",
+ "-Xmx2g",
+ "-Xms2g",
+ "-XX:+AlwaysPreTouch"
+ })
+ public long longValuesPanamaProvider() throws IOException {
+ return readBatch();
+ }
+
+ private long readBatch() throws IOException {
+ final int step = accessPattern.equals("contiguous") ? 1 : 2;
+ final int maxStart = docCount - (batchSize - 1) * step - 1;
+ if (nextStart > maxStart) {
+ nextStart = 0;
+ }
+ for (int i = 0, doc = nextStart; i < batchSize; i++, doc += step) {
+ docs[i] = doc;
+ }
+ nextStart += batchSize * step;
+
+ values.longValues(batchSize, docs, valueBuffer, 0);
+ long checksum = 0;
+ for (int i = 0; i < batchSize; i++) {
+ checksum += valueBuffer[i];
+ }
+ return checksum;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
index 1abf0c79818..5cd3e7afac3 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
@@ -21,6 +21,7 @@ import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_IND
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BaseTermsEnum;
@@ -462,6 +463,10 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
DOC_VALUES_RANGE_SUPPORT =
org.apache.lucene.internal.vectorization.VectorizationProvider.getInstance()
.getDocValuesRangeSupport();
+ private static final org.apache.lucene.internal.vectorization.DocValuesBulkDecodeSupport
+ DOC_VALUES_BULK_DECODE_SUPPORT =
+ org.apache.lucene.internal.vectorization.VectorizationProvider.getInstance()
+ .getDocValuesBulkDecodeSupport();
static void rangeIntoBitSet(
org.apache.lucene.util.LongValues values,
@@ -475,6 +480,73 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
values, fromDoc, toDoc, minValue, maxValue, bitSet, offset);
}
+ private static boolean canBulkDecodeByteAligned(NumericEntry entry) {
+ return entry.blockShift < 0 && entry.bitsPerValue > 0 && (entry.bitsPerValue & 0x07) == 0;
+ }
+
+ private static boolean isContiguous(int size, int[] docs, int docsOffset) {
+ return size == 0 || docs[docsOffset + size - 1] - docs[docsOffset] == size - 1;
+ }
+
+ private static int paddingBytesNeededForBulkDecode(int bitsPerValue) {
+ if (bitsPerValue == 24) {
+ return 1;
+ } else if (bitsPerValue == 40 || bitsPerValue == 48 || bitsPerValue == 56) {
+ return Long.BYTES - bitsPerValue / Byte.SIZE;
+ }
+ return 0;
+ }
+
+ private static byte[] bulkDecodeByteAlignedValues(
+ RandomAccessInput slice,
+ NumericEntry entry,
+ int size,
+ int[] docs,
+ int docsOffset,
+ long[] values,
+ int valuesOffset,
+ byte[] bytes)
+ throws IOException {
+ if (canBulkDecodeByteAligned(entry) == false || isContiguous(size, docs, docsOffset) == false) {
+ return null;
+ }
+
+ final int bytesPerValue = entry.bitsPerValue / Byte.SIZE;
+ final long byteCountLong = (long) size * bytesPerValue;
+ if (byteCountLong > Integer.MAX_VALUE) {
+ return null;
+ }
+ final int byteCount = (int) byteCountLong;
+ final int readByteCount =
+ byteCount == 0 ? 0 : byteCount + paddingBytesNeededForBulkDecode(entry.bitsPerValue);
+ final long offset = byteCount == 0 ? 0 : (long) docs[docsOffset] * bytesPerValue;
+ if (offset + readByteCount > slice.length()) {
+ return null;
+ }
+ if (bytes.length < readByteCount) {
+ bytes = new byte[readByteCount];
+ }
+ if (byteCount != 0) {
+ slice.readBytes(offset, bytes, 0, readByteCount);
+ DOC_VALUES_BULK_DECODE_SUPPORT.decodeByteAligned(
+ bytes, 0, entry.bitsPerValue, values, valuesOffset, size);
+ }
+ return bytes;
+ }
+
+ private static void applyTable(long[] values, int valuesOffset, long[] table, int size) {
+ for (int i = valuesOffset, end = valuesOffset + size; i < end; i++) {
+ values[i] = table[(int) values[i]];
+ }
+ }
+
+ private static void applyGcdDelta(
+ long[] values, int valuesOffset, long mul, long delta, int size) {
+ for (int i = valuesOffset, end = valuesOffset + size; i < end; i++) {
+ values[i] = mul * values[i] + delta;
+ }
+ }
+
private static class NumericEntry {
long[] table;
int blockShift;
@@ -667,6 +739,21 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
public long longValue() throws IOException {
return entry.minValue;
}
+
+ @Override
+ public void longValues(
+ int size,
+ int[] docs,
+ int docsOffset,
+ long[] values,
+ int valuesOffset,
+ long defaultValue)
+ throws IOException {
+ Arrays.fill(values, valuesOffset, valuesOffset + size, entry.minValue);
+ if (size != 0) {
+ doc = docs[docsOffset + size - 1];
+ }
+ }
};
} else {
final RandomAccessInput slice =
@@ -685,6 +772,20 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
public long longValue() throws IOException {
return vBPVReader.getLongValue(doc);
}
+
+ @Override
+ public void longValues(
+ int size,
+ int[] docs,
+ int docsOffset,
+ long[] values,
+ int valuesOffset,
+ long defaultValue)
+ throws IOException {
+ // Delegate to help performance: when the super call inlines, calls to
+ // #advanceExact/#longValue become monomorphic.
+ super.longValues(size, docs, docsOffset, values, valuesOffset, defaultValue);
+ }
};
} else {
final LongValues values =
@@ -692,19 +793,68 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
if (entry.table != null) {
final long[] table = entry.table;
return new DenseNumericDocValues(maxDoc) {
+ private byte[] bulkBytes = new byte[0];
+
@Override
public long longValue() throws IOException {
return table[(int) values.get(doc)];
}
+
+ @Override
+ public void longValues(
+ int size,
+ int[] docs,
+ int docsOffset,
+ long[] values,
+ int valuesOffset,
+ long defaultValue)
+ throws IOException {
+ byte[] bytes =
+ bulkDecodeByteAlignedValues(
+ slice, entry, size, docs, docsOffset, values, valuesOffset, bulkBytes);
+ if (bytes == null) {
+ super.longValues(size, docs, docsOffset, values, valuesOffset, defaultValue);
+ } else {
+ applyTable(values, valuesOffset, table, size);
+ bulkBytes = bytes;
+ if (size != 0) {
+ doc = docs[docsOffset + size - 1];
+ }
+ }
+ }
};
} else if (entry.gcd == 1 && entry.minValue == 0) {
// Common case for ordinals, which are encoded as numerics
return new DenseNumericDocValues(maxDoc) {
+ private byte[] bulkBytes = new byte[0];
+
@Override
public long longValue() throws IOException {
return values.get(doc);
}
+ @Override
+ public void longValues(
+ int size,
+ int[] docs,
+ int docsOffset,
+ long[] values,
+ int valuesOffset,
+ long defaultValue)
+ throws IOException {
+ byte[] bytes =
+ bulkDecodeByteAlignedValues(
+ slice, entry, size, docs, docsOffset, values, valuesOffset, bulkBytes);
+ if (bytes == null) {
+ super.longValues(size, docs, docsOffset, values, valuesOffset, defaultValue);
+ } else {
+ bulkBytes = bytes;
+ if (size != 0) {
+ doc = docs[docsOffset + size - 1];
+ }
+ }
+ }
+
@Override
public void rangeIntoBitSet(
int fromDoc,
@@ -722,11 +872,36 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
final long mul = entry.gcd;
final long delta = entry.minValue;
return new DenseNumericDocValues(maxDoc) {
+ private byte[] bulkBytes = new byte[0];
+
@Override
public long longValue() throws IOException {
return mul * values.get(doc) + delta;
}
+ @Override
+ public void longValues(
+ int size,
+ int[] docs,
+ int docsOffset,
+ long[] values,
+ int valuesOffset,
+ long defaultValue)
+ throws IOException {
+ byte[] bytes =
+ bulkDecodeByteAlignedValues(
+ slice, entry, size, docs, docsOffset, values, valuesOffset, bulkBytes);
+ if (bytes == null) {
+ super.longValues(size, docs, docsOffset, values, valuesOffset, defaultValue);
+ } else {
+ applyGcdDelta(values, valuesOffset, mul, delta, size);
+ bulkBytes = bytes;
+ if (size != 0) {
+ doc = docs[docsOffset + size - 1];
+ }
+ }
+ }
+
@Override
public void rangeIntoBitSet(
int fromDoc,
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
index 39d0131d93e..1b83d727ec3 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValues.java
@@ -81,15 +81,32 @@ public abstract class NumericDocValues extends DocValuesIterator {
*/
public void longValues(int size, int[] docs, long[] values, long defaultValue)
throws IOException {
- for (int i = 0; i < size; ++i) {
- int doc = docs[i];
+ longValues(size, docs, 0, values, 0, defaultValue);
+ }
+
+ /**
+ * Offset-aware variant of {@link #longValues(int, int[], long[], long)}. Reads {@code size} doc
+ * IDs starting at {@code docs[docsOffset]} and writes the corresponding values starting at {@code
+ * values[valuesOffset]}. This follows the same convention as {@link System#arraycopy}.
+ *
+ * @param size the number of values to retrieve
+ * @param docs the buffer of doc IDs whose values should be looked up
+ * @param docsOffset first position in {@code docs} to read
+ * @param values the buffer of values to fill
+ * @param valuesOffset first position in {@code values} to write
+ * @param defaultValue the value to put in the buffer when a document doesn't have a value
+ */
+ public void longValues(
+ int size, int[] docs, int docsOffset, long[] values, int valuesOffset, long defaultValue)
+ throws IOException {
+ for (int di = docsOffset, vi = valuesOffset, end = docsOffset + size; di < end; di++, vi++) {
long value;
- if (advanceExact(doc)) {
+ if (advanceExact(docs[di])) {
value = longValue();
} else {
value = defaultValue;
}
- values[i] = value;
+ values[vi] = value;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultDocValuesBulkDecodeSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultDocValuesBulkDecodeSupport.java
new file mode 100644
index 00000000000..0fe14d20ca9
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultDocValuesBulkDecodeSupport.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.internal.vectorization;
+
+import org.apache.lucene.util.BitUtil;
+
+/** Scalar (non-SIMD) implementation of {@link DocValuesBulkDecodeSupport}. */
+final class DefaultDocValuesBulkDecodeSupport implements DocValuesBulkDecodeSupport {
+
+ static final DefaultDocValuesBulkDecodeSupport INSTANCE = new DefaultDocValuesBulkDecodeSupport();
+
+ private static final int BYTES_PER_24_BIT_VALUE = 24 / Byte.SIZE;
+ private static final int BYTES_PER_40_BIT_VALUE = 40 / Byte.SIZE;
+ private static final int BYTES_PER_48_BIT_VALUE = 48 / Byte.SIZE;
+ private static final int BYTES_PER_56_BIT_VALUE = 56 / Byte.SIZE;
+
+ private DefaultDocValuesBulkDecodeSupport() {}
+
+ @Override
+ public void decodeByteAligned(
+ byte[] bytes, int bytesOffset, int bitsPerValue, long[] values, int valuesOffset, int count) {
+ switch (bitsPerValue) {
+ case Byte.SIZE -> decode8(bytes, bytesOffset, values, valuesOffset, count);
+ case Short.SIZE -> decode16(bytes, bytesOffset, values, valuesOffset, count);
+ case 24 -> decode24(bytes, bytesOffset, values, valuesOffset, count);
+ case Integer.SIZE -> decode32(bytes, bytesOffset, values, valuesOffset, count);
+ case 40 -> decode40(bytes, bytesOffset, values, valuesOffset, count);
+ case 48 -> decode48(bytes, bytesOffset, values, valuesOffset, count);
+ case 56 -> decode56(bytes, bytesOffset, values, valuesOffset, count);
+ case Long.SIZE -> decode64(bytes, bytesOffset, values, valuesOffset, count);
+ default -> throw new IllegalArgumentException("unsupported bitsPerValue: " + bitsPerValue);
+ }
+ }
+
+ private static void decode8(
+ byte[] bytes, int bytesOffset, long[] values, int valuesOffset, int count) {
+ for (int vi = valuesOffset, bi = bytesOffset, end = valuesOffset + count;
+ vi < end;
+ vi++, bi++) {
+ values[vi] = Byte.toUnsignedLong(bytes[bi]);
+ }
+ }
+
+ private static void decode16(
+ byte[] bytes, int bytesOffset, long[] values, int valuesOffset, int count) {
+ for (int vi = valuesOffset, bi = bytesOffset, end = valuesOffset + count;
+ vi < end;
+ vi++, bi += Short.BYTES) {
+ values[vi] = Short.toUnsignedLong((short) BitUtil.VH_LE_SHORT.get(bytes, bi));
+ }
+ }
+
+ private static void decode24(
+ byte[] bytes, int bytesOffset, long[] values, int valuesOffset, int count) {
+ for (int vi = valuesOffset, bi = bytesOffset, end = valuesOffset + count;
+ vi < end;
+ vi++, bi += BYTES_PER_24_BIT_VALUE) {
+ values[vi] = ((int) BitUtil.VH_LE_INT.get(bytes, bi)) & 0xFFFFFFL;
+ }
+ }
+
+ private static void decode32(
+ byte[] bytes, int bytesOffset, long[] values, int valuesOffset, int count) {
+ for (int vi = valuesOffset, bi = bytesOffset, end = valuesOffset + count;
+ vi < end;
+ vi++, bi += Integer.BYTES) {
+ values[vi] = Integer.toUnsignedLong((int) BitUtil.VH_LE_INT.get(bytes, bi));
+ }
+ }
+
+ private static void decode40(
+ byte[] bytes, int bytesOffset, long[] values, int valuesOffset, int count) {
+ for (int vi = valuesOffset, bi = bytesOffset, end = valuesOffset + count;
+ vi < end;
+ vi++, bi += BYTES_PER_40_BIT_VALUE) {
+ values[vi] = ((long) BitUtil.VH_LE_LONG.get(bytes, bi)) & 0xFFFFFFFFFFL;
+ }
+ }
+
+ private static void decode48(
+ byte[] bytes, int bytesOffset, long[] values, int valuesOffset, int count) {
+ for (int vi = valuesOffset, bi = bytesOffset, end = valuesOffset + count;
+ vi < end;
+ vi++, bi += BYTES_PER_48_BIT_VALUE) {
+ values[vi] = ((long) BitUtil.VH_LE_LONG.get(bytes, bi)) & 0xFFFFFFFFFFFFL;
+ }
+ }
+
+ private static void decode56(
+ byte[] bytes, int bytesOffset, long[] values, int valuesOffset, int count) {
+ for (int vi = valuesOffset, bi = bytesOffset, end = valuesOffset + count;
+ vi < end;
+ vi++, bi += BYTES_PER_56_BIT_VALUE) {
+ values[vi] = ((long) BitUtil.VH_LE_LONG.get(bytes, bi)) & 0xFFFFFFFFFFFFFFL;
+ }
+ }
+
+ private static void decode64(
+ byte[] bytes, int bytesOffset, long[] values, int valuesOffset, int count) {
+ for (int vi = valuesOffset, bi = bytesOffset, end = valuesOffset + count;
+ vi < end;
+ vi++, bi += Long.BYTES) {
+ values[vi] = (long) BitUtil.VH_LE_LONG.get(bytes, bi);
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorizationProvider.java
index 0394e195966..4d38c3f8da0 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorizationProvider.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorizationProvider.java
@@ -55,4 +55,9 @@ final class DefaultVectorizationProvider extends VectorizationProvider {
public DocValuesRangeSupport getDocValuesRangeSupport() {
return DefaultDocValuesRangeSupport.INSTANCE;
}
+
+ @Override
+ public DocValuesBulkDecodeSupport getDocValuesBulkDecodeSupport() {
+ return DefaultDocValuesBulkDecodeSupport.INSTANCE;
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesBulkDecodeSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesBulkDecodeSupport.java
new file mode 100644
index 00000000000..e787510cd42
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DocValuesBulkDecodeSupport.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.internal.vectorization;
+
+import org.apache.lucene.util.LongValues;
+
+/**
+ * Interface for SIMD-accelerated doc values bulk decode operations.
+ *
+ * <p>Implementations decode byte-aligned packed numeric values into a {@code long[]} destination.
+ * The default scalar implementation is used when the Panama Vector API is unavailable; a
+ * SIMD-accelerated implementation is used otherwise.
+ *
+ * @lucene.internal
+ */
+public interface DocValuesBulkDecodeSupport {
+
+ /**
+ * Decodes {@code count} byte-aligned packed values from {@code bytes} into {@code values}.
+ *
+ * @param bytes source bytes encoded like {@link LongValues} produced by packed {@code
+ * DirectReader}
+ * @param bytesOffset first byte to read
+ * @param bitsPerValue number of bits per value, must be a multiple of 8
+ * @param values destination values
+ * @param valuesOffset first destination slot
+ * @param count number of values to decode
+ */
+ void decodeByteAligned(
+ byte[] bytes, int bytesOffset, int bitsPerValue, long[] values, int valuesOffset, int count);
+}
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
index 2bd1c2c3b1e..b6feef9358c 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
@@ -122,6 +122,13 @@ public abstract class VectorizationProvider {
*/
public abstract DocValuesRangeSupport getDocValuesRangeSupport();
+ /**
+ * Returns a {@link DocValuesBulkDecodeSupport} instance for bulk numeric value decode. The
+ * returned instance uses SIMD when available (Panama Vector API), falling back to a scalar loop
+ * otherwise.
+ */
+ public abstract DocValuesBulkDecodeSupport getDocValuesBulkDecodeSupport();
+
// *** Lookup mechanism: ***
private static final Logger LOG = Logger.getLogger(VectorizationProvider.class.getName());
diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/NativeVectorizationProvider.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/NativeVectorizationProvider.java
index 7234b0dbca9..1233cc2f89f 100644
--- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/NativeVectorizationProvider.java
+++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/NativeVectorizationProvider.java
@@ -81,4 +81,9 @@ final class NativeVectorizationProvider extends VectorizationProvider {
public DocValuesRangeSupport getDocValuesRangeSupport() {
return delegateVectorUtilProvider.getDocValuesRangeSupport();
}
+
+ @Override
+ public DocValuesBulkDecodeSupport getDocValuesBulkDecodeSupport() {
+ return delegateVectorUtilProvider.getDocValuesBulkDecodeSupport();
+ }
}
diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesBulkDecodeSupport.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesBulkDecodeSupport.java
new file mode 100644
index 00000000000..a2cbdde6b80
--- /dev/null
+++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaDocValuesBulkDecodeSupport.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.internal.vectorization;
+
+import java.nio.ByteOrder;
+import jdk.incubator.vector.ByteVector;
+import jdk.incubator.vector.VectorSpecies;
+
+/** Panama Vector API implementation of {@link DocValuesBulkDecodeSupport}. */
+final class PanamaDocValuesBulkDecodeSupport implements DocValuesBulkDecodeSupport {
+
+ static final PanamaDocValuesBulkDecodeSupport INSTANCE = new PanamaDocValuesBulkDecodeSupport();
+
+ private static final VectorSpecies<Byte> BYTE_SPECIES = ByteVector.SPECIES_PREFERRED;
+
+ private PanamaDocValuesBulkDecodeSupport() {}
+
+ @Override
+ public void decodeByteAligned(
+ byte[] bytes, int bytesOffset, int bitsPerValue, long[] values, int valuesOffset, int count) {
+ if (bitsPerValue != Long.SIZE
+ || ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN
+ || BYTE_SPECIES.vectorByteSize() < 32) {
+ DefaultDocValuesBulkDecodeSupport.INSTANCE.decodeByteAligned(
+ bytes, bytesOffset, bitsPerValue, values, valuesOffset, count);
+ return;
+ }
+
+ final int valuesPerVector = BYTE_SPECIES.vectorByteSize() / Long.BYTES;
+ final int loopBound = count - count % valuesPerVector;
+ int i = 0;
+ for (; i < loopBound; i += valuesPerVector) {
+ ByteVector.fromArray(BYTE_SPECIES, bytes, bytesOffset + i * Long.BYTES)
+ .reinterpretAsLongs()
+ .intoArray(values, valuesOffset + i);
+ }
+ if (i < count) {
+ DefaultDocValuesBulkDecodeSupport.INSTANCE.decodeByteAligned(
+ bytes, bytesOffset + i * Long.BYTES, bitsPerValue, values, valuesOffset + i, count - i);
+ }
+ }
+}
diff --git a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
index 2e5917b9f9b..f52bd6a89ec 100644
--- a/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
+++ b/lucene/core/src/java25/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java
@@ -98,4 +98,9 @@ final class PanamaVectorizationProvider extends VectorizationProvider {
public DocValuesRangeSupport getDocValuesRangeSupport() {
return PanamaDocValuesRangeSupport.INSTANCE;
}
+
+ @Override
+ public DocValuesBulkDecodeSupport getDocValuesBulkDecodeSupport() {
+ return PanamaDocValuesBulkDecodeSupport.INSTANCE;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
index 50cb8b7aab3..d816c250d1e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
@@ -176,6 +176,109 @@ public class TestLucene90DocValuesFormat extends BaseCompressingDocValuesFormatT
}
}
+ public void testDenseNumericLongValuesBulkFetch() throws Exception {
+ doTestDenseNumericLongValuesBulkFetch(8);
+ doTestDenseNumericLongValuesBulkFetch(16);
+ doTestDenseNumericLongValuesBulkFetch(24);
+ doTestDenseNumericLongValuesBulkFetch(32);
+ doTestDenseNumericLongValuesBulkFetch(40);
+ doTestDenseNumericLongValuesBulkFetch(48);
+ doTestDenseNumericLongValuesBulkFetch(56);
+ doTestDenseNumericLongValuesBulkFetch(64);
+ }
+
+ private void doTestDenseNumericLongValuesBulkFetch(int bitsPerValue) throws Exception {
+ final int numDocs = 512;
+ final long[] expected = new long[numDocs];
+ final long mask = bitsPerValue == 64 ? -1L : (1L << bitsPerValue) - 1;
+ for (int i = 0; i < numDocs; i++) {
+ expected[i] = bitsPerValue == 64 ? random().nextLong() : random().nextLong() & mask;
+ }
+
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
+ conf.setMergeScheduler(new SerialMergeScheduler());
+ IndexWriter writer = new IndexWriter(dir, conf);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("numeric", expected[i]));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ NumericDocValues values = reader.leaves().get(0).reader().getNumericDocValues("numeric");
+ NumericDocValues singleValues =
+ reader.leaves().get(0).reader().getNumericDocValues("numeric");
+ assertBulkLongValues(bitsPerValue, values, singleValues, expected, 0, 128, 1);
+ assertBulkLongValues(bitsPerValue, values, singleValues, expected, 17, 128, 1);
+ assertBulkLongValues(bitsPerValue, values, singleValues, expected, 0, 128, 2);
+ }
+
+ writer.close();
+ dir.close();
+ }
+
+ private static void assertBulkLongValues(
+ int bitsPerValue,
+ NumericDocValues values,
+ NumericDocValues singleValues,
+ long[] expected,
+ int startDoc,
+ int size,
+ int docStep)
+ throws IOException {
+ int[] docs = new int[size];
+ long[] actual = new long[size];
+ for (int i = 0; i < size; i++) {
+ docs[i] = startDoc + i * docStep;
+ }
+
+ values.longValues(size, docs, actual, 0);
+ if (size != 0) {
+ assertEquals(docs[size - 1], values.docID());
+ }
+ for (int i = 0; i < size; i++) {
+ assertTrue(singleValues.advanceExact(docs[i]));
+ long singleValue = singleValues.longValue();
+ assertEquals(
+ "bitsPerValue="
+ + bitsPerValue
+ + " doc="
+ + docs[i]
+ + " startDoc="
+ + startDoc
+ + " docStep="
+ + docStep
+ + " indexedValue="
+ + expected[docs[i]],
+ singleValue,
+ actual[i]);
+ assertEquals(
+ "bitsPerValue=" + bitsPerValue + " doc=" + docs[i], expected[docs[i]], singleValue);
+ }
+
+ // Verify offset-aware variant produces identical results
+ if (size > 2) {
+ int docsOffset = 1;
+ int sliceSize = size - 2;
+ long[] offsetActual = new long[sliceSize + 4];
+ int valuesOffset = 2;
+ Arrays.fill(offsetActual, Long.MIN_VALUE);
+ values.longValues(sliceSize, docs, docsOffset, offsetActual, valuesOffset, 0);
+ for (int i = 0; i < sliceSize; i++) {
+ assertEquals(
+ "offset variant mismatch at i=" + i + " bpv=" + bitsPerValue,
+ actual[docsOffset + i],
+ offsetActual[valuesOffset + i]);
+ }
+ // sentinel positions should be untouched
+ assertEquals(Long.MIN_VALUE, offsetActual[0]);
+ assertEquals(Long.MIN_VALUE, offsetActual[1]);
+ assertEquals(Long.MIN_VALUE, offsetActual[valuesOffset + sliceSize]);
+ }
+ }
+
private void doTestSparseDocValuesVsStoredFields() throws Exception {
final long[] values = new long[TestUtil.nextInt(random(), 1, 500)];
for (int i = 0; i < values.length; ++i) {