You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ti...@apache.org on 2026/06/12 11:03:18 UTC
(lucene) branch branch_10x updated: Add dense indexing path for BinaryColumn (#16227)
This is an automated email from the ASF dual-hosted git repository.
Tim-Brooks pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 7a0042427ff Add dense indexing path for BinaryColumn (#16227)
7a0042427ff is described below
commit 7a0042427ff542a7857a97e1d1182d5f90425302
Author: Tim Brooks <ti...@timbrooks.org>
AuthorDate: Fri Jun 12 12:57:36 2026 +0200
Add dense indexing path for BinaryColumn (#16227)
When a BinaryColumn reports Density.DENSE, IndexingChain now
bulk-consumes values from a BytesRefValuesCursor instead of iterating
per-doc tuples, mirroring the dense LongColumn/DictionaryColumn paths.
The points pass goes through PointValuesWriter.addDenseNDValues, which
stages fixed-width N-D records into the scratch buffer in chunks and
drains each with a single writeBytes, rather than one writeBytes per
value. Contiguous columns can override BytesRefValuesCursor.
fillPackedPoints to fill a chunk with a single arraycopy.
Replace the per-writer 1 KB densePointsBuffer with a chain-wide
SharedIndexingScratch (4 KB BYTES_SCRATCH_SIZE). RAM is tracked via
the chain Counter.
---
lucene/CHANGES.txt | 4 +-
.../lucene/document/column/BinaryColumn.java | 11 ++
.../document/column/BytesRefValuesCursor.java | 81 ++++++++
.../org/apache/lucene/index/IndexingChain.java | 57 +++++-
.../org/apache/lucene/index/PointValuesWriter.java | 71 ++++---
.../apache/lucene/index/SharedIndexingScratch.java | 55 ++++++
.../document/column/ColumnBatchTestUtil.java | 119 ++++++++++++
.../column/TestColumnBatchBinaryColumn.java | 208 +++++++++++++++++++++
.../document/column/TestColumnBatchLongColumn.java | 8 +-
9 files changed, 583 insertions(+), 31 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f2af01b0d84..60406d2e946 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -38,10 +38,12 @@ API Changes
* GITHUB#16062: Add FlatFieldVectorsWriter.asKnnVectorValues to return directly a KnnVectorValues "view" over
vectors to be written. (Lorenzo Dematte)
-* GITHUB#16101 Add DictionaryColumn for SORTED / SORTED_SET docvalues using experimental column api (Tim Brooks)
+* GITHUB#16101: Add DictionaryColumn for SORTED / SORTED_SET docvalues using experimental column api (Tim Brooks)
* GITHUB#16104: Add public constructors and a public getSearchStrategy() to FloatVectorSimilarityQuery, ByteVectorSimilarityQuery and VectorSimilarityCollector that accept a KnnSearchStrategy. (Sasilekha R)
+* GITHUB#16227: Add dense indexing path for BinaryColumn while using experimental column api (Tim Brooks)
+
New Features
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.java b/lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.java
index b26c5b495fc..2c66c8be5f9 100644
--- a/lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.java
+++ b/lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.java
@@ -54,4 +54,15 @@ public abstract class BinaryColumn extends Column {
/** Returns a fresh tuple cursor starting at the beginning of the batch. */
public abstract ObjectTupleCursor<BytesRef> tuples();
+
+ /**
+ * Returns a fresh values cursor iterating dense binary values for doc-ids {@code [0, numDocs)}.
+ * Must be overridden when {@link Column#density()} is {@link Column.Density#DENSE DENSE}; the
+ * default implementation throws {@link UnsupportedOperationException} and is never called for
+ * {@link Column.Density#SPARSE SPARSE} columns.
+ */
+ public BytesRefValuesCursor values() {
+ throw new UnsupportedOperationException(
+ "values() requires density() == DENSE for column \"" + name() + "\"");
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/document/column/BytesRefValuesCursor.java b/lucene/core/src/java/org/apache/lucene/document/column/BytesRefValuesCursor.java
new file mode 100644
index 00000000000..935d5dd1e3b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/document/column/BytesRefValuesCursor.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document.column;
+
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A values cursor over a dense {@link BinaryColumn}. The cursor produces exactly {@link #size()}
+ * values for consecutive batch-local doc-ids starting at 0, one per call to {@link #nextValue()}.
+ *
+ * <p>Implementations must throw an exception if {@link #nextValue()} is called more than {@link
+ * #size()} times. The returned {@link BytesRef} is valid only until the next call to {@link
+ * #nextValue()}.
+ *
+ * <p>Combined consumption across {@link #nextValue()} and {@link #fillPackedPoints} must not exceed
+ * {@link #size()}.
+ *
+ * @lucene.experimental
+ */
+public abstract class BytesRefValuesCursor {
+
+ private final int size;
+
+ /**
+ * Creates a cursor that will produce exactly {@code size} values, one per batch-local doc-id in
+ * {@code [0, size)}. {@code size} is fixed for the cursor's lifetime and must equal the dense
+ * column's {@code numDocs}.
+ *
+ * <p>Lucene's internal indexing paths will not consume past {@code size} across {@link
+ * #nextValue()} and {@link #fillPackedPoints}. Defensive throws on overrun are still encouraged
+ * to catch misuse from external callers.
+ */
+ protected BytesRefValuesCursor(int size) {
+ this.size = size;
+ }
+
+ /** Total number of values this cursor will produce. */
+ public final int size() {
+ return size;
+ }
+
+ /**
+ * Returns the next {@link BytesRef} value. Must not be called more than {@link #size()} times.
+ */
+ public abstract BytesRef nextValue();
+
+ /**
+ * Bulk-fill {@code length} fixed-width packed point records into {@code dst} starting at byte
+ * {@code offset}, advancing the cursor by {@code length}. Each value must be exactly {@code
+ * width} bytes and is passed through unchanged (no sortable encoding). Combined consumption
+ * across {@link #nextValue()} and this method must not exceed {@link #size()}.
+ *
+ * <p>The default implementation calls {@link #nextValue()} per value and validates each length.
+ * Override when the backing data is optimizable (e.g. a contiguous packed array); such overrides
+ * take responsibility for the width contract.
+ */
+ public void fillPackedPoints(byte[] dst, int offset, int length, int width) {
+ for (int i = 0; i < length; i++) {
+ BytesRef v = nextValue();
+ if (v.length != width) {
+ throw new IllegalArgumentException(
+ "dense point value has length=" + v.length + " but should be " + width);
+ }
+ System.arraycopy(v.bytes, v.offset, dst, offset + i * width, width);
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index 1fd23f40f2a..8eeffedcccb 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -44,6 +44,7 @@ import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.document.column.BinaryColumn;
+import org.apache.lucene.document.column.BytesRefValuesCursor;
import org.apache.lucene.document.column.Column;
import org.apache.lucene.document.column.ColumnBatch;
import org.apache.lucene.document.column.ColumnFieldAdapter;
@@ -86,6 +87,8 @@ final class IndexingChain implements Accountable {
final TermsHash termsHash;
// Shared pool for doc-value terms
final ByteBlockPool docValuesBytePool;
+ // Shared scratch buffers for dense points encoding
+ final SharedIndexingScratch sharedIndexingScratch;
// Writes stored fields
final StoredFieldsConsumer storedFieldsConsumer;
final VectorValuesConsumer vectorValuesConsumer;
@@ -155,6 +158,7 @@ final class IndexingChain implements Accountable {
new FreqProxTermsWriter(
intBlockAllocator, byteBlockAllocator, bytesUsed, termVectorsWriter);
docValuesBytePool = new ByteBlockPool(byteBlockAllocator);
+ sharedIndexingScratch = new SharedIndexingScratch(bytesUsed);
if (indexWriterConfig.getParentField() != null) {
this.parentField = new NumericDocValuesField(indexWriterConfig.getParentField(), -1);
parentPf = getOrAddPerField(this.parentField.name());
@@ -1091,6 +1095,12 @@ final class IndexingChain implements Accountable {
throws IOException {
final DocValuesType dvType = fieldType.docValuesType();
final boolean hasPoints = fieldType.pointDimensionCount() != 0;
+
+ if (column.density() == Column.Density.DENSE) {
+ processDenseBinaryColumn(baseDocID, numDocs, column, pf, dvType, hasPoints);
+ return;
+ }
+
final PointValuesWriter pointWriter = hasPoints ? pf.pointValuesWriter : null;
final ObjectTupleCursor<BytesRef> cursor = column.tuples();
@@ -1152,6 +1162,51 @@ final class IndexingChain implements Accountable {
}
}
+ private static void processDenseBinaryColumn(
+ int baseDocID,
+ int numDocs,
+ BinaryColumn column,
+ PerField pf,
+ DocValuesType dvType,
+ boolean hasPoints)
+ throws IOException {
+ // DV pass first: dense values cursor
+ if (dvType != DocValuesType.NONE) {
+ BytesRefValuesCursor dvCursor = column.values();
+ ColumnValidation.checkDenseCount(column, dvCursor.size(), numDocs);
+ switch (dvType) {
+ case BINARY -> {
+ BinaryDocValuesWriter writer = (BinaryDocValuesWriter) pf.docValuesWriter;
+ for (int i = 0; i < dvCursor.size(); i++) {
+ writer.addValue(baseDocID + i, dvCursor.nextValue());
+ }
+ }
+ case SORTED -> {
+ SortedDocValuesWriter writer = (SortedDocValuesWriter) pf.docValuesWriter;
+ for (int i = 0; i < dvCursor.size(); i++) {
+ writer.addValue(baseDocID + i, dvCursor.nextValue());
+ }
+ }
+ case SORTED_SET -> {
+ SortedSetDocValuesWriter writer = (SortedSetDocValuesWriter) pf.docValuesWriter;
+ for (int i = 0; i < dvCursor.size(); i++) {
+ writer.addValue(baseDocID + i, dvCursor.nextValue());
+ }
+ }
+ // $CASES-OMITTED$
+ default ->
+ throw new IllegalArgumentException(
+ "BinaryColumn \"" + column.name() + "\" has incompatible docValuesType: " + dvType);
+ }
+ }
+ if (hasPoints) {
+ // Points pass: fresh dense values cursor → bulk ND points add.
+ BytesRefValuesCursor pc = column.values();
+ ColumnValidation.checkDenseCount(column, pc.size(), numDocs);
+ pf.pointValuesWriter.addDenseNDValues(baseDocID, pc);
+ }
+ }
+
private static void processDictionaryColumn(
int baseDocID,
int numDocs,
@@ -1300,7 +1355,7 @@ final class IndexingChain implements Accountable {
throw new AssertionError("unrecognized DocValues.Type: " + dvType);
}
if (fi.getPointDimensionCount() != 0) {
- pf.pointValuesWriter = new PointValuesWriter(bytesUsed, fi);
+ pf.pointValuesWriter = new PointValuesWriter(bytesUsed, fi, sharedIndexingScratch);
}
if (fi.getVectorDimension() != 0) {
try {
diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
index 5443d2919b7..4bff39999cc 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java
@@ -20,12 +20,14 @@ import java.io.IOException;
import org.apache.lucene.codecs.MutablePointTree;
import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.document.column.BytesRefValuesCursor;
import org.apache.lucene.document.column.LongValuesCursor;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.bkd.BKDConfig;
/** Buffers up pending byte[][] value(s) per doc, then flushes when segment flushes. */
class PointValuesWriter {
@@ -33,26 +35,33 @@ class PointValuesWriter {
private final PagedBytes bytes;
private final DataOutput bytesOut;
private final Counter iwBytesUsed;
+ private final SharedIndexingScratch sharedScratch;
private int[] docIDs;
private int numPoints;
private int numDocs;
private int lastDocID = -1;
private final int packedBytesLength;
- /**
- * Lazily-allocated 1 KB scratch used by the dense bulk paths to stage encoded points before
- * pushing to {@link #bytesOut} via a single {@code writeBytes}.
- */
- private static final int POINTS_BUFFER_BYTES = 1024;
-
- private static final int POINTS_BUFFER_INT_VALUES = POINTS_BUFFER_BYTES / Integer.BYTES;
- private static final int POINTS_BUFFER_LONG_VALUES = POINTS_BUFFER_BYTES / Long.BYTES;
-
- private byte[] densePointsBuffer;
+ private static final int POINTS_BUFFER_INT_VALUES =
+ SharedIndexingScratch.BYTES_SCRATCH_SIZE / Integer.BYTES;
+ private static final int POINTS_BUFFER_LONG_VALUES =
+ SharedIndexingScratch.BYTES_SCRATCH_SIZE / Long.BYTES;
+
+ static {
+ // If Lucene ever supports packed points > BYTES_SCRATCH_SIZE either the buffer needs to be
+ // increased or a fallback indexing code-path bypassing the buffer needs to be added.
+ assert SharedIndexingScratch.BYTES_SCRATCH_SIZE
+ >= PointValues.MAX_NUM_BYTES * BKDConfig.MAX_DIMS
+ : "BYTES_SCRATCH_SIZE="
+ + SharedIndexingScratch.BYTES_SCRATCH_SIZE
+ + " must be >= MAX_NUM_BYTES * MAX_DIMS="
+ + (PointValues.MAX_NUM_BYTES * BKDConfig.MAX_DIMS);
+ }
- PointValuesWriter(Counter bytesUsed, FieldInfo fieldInfo) {
+ PointValuesWriter(Counter bytesUsed, FieldInfo fieldInfo, SharedIndexingScratch sharedScratch) {
this.fieldInfo = fieldInfo;
this.iwBytesUsed = bytesUsed;
+ this.sharedScratch = sharedScratch;
this.bytes = new PagedBytes(12);
bytesOut = bytes.getDataOutput();
docIDs = new int[16];
@@ -98,8 +107,8 @@ class PointValuesWriter {
if (size == 0) {
return;
}
- final long ramBefore = reserveDense1D(firstDocID, size);
- final byte[] buffer = pointsDenseBuffer();
+ final long ramBefore = reserveDenseRange(firstDocID, size);
+ final byte[] buffer = sharedScratch.bytesScratch();
int remaining = size;
while (remaining > 0) {
int chunk = Math.min(POINTS_BUFFER_INT_VALUES, remaining);
@@ -107,7 +116,7 @@ class PointValuesWriter {
bytesOut.writeBytes(buffer, 0, chunk * Integer.BYTES);
remaining -= chunk;
}
- commitDense1D(firstDocID, size, ramBefore);
+ commitDenseRange(firstDocID, size, ramBefore);
}
void addDense1DLongValues(int firstDocID, LongValuesCursor cursor) throws IOException {
@@ -116,23 +125,35 @@ class PointValuesWriter {
if (size == 0) {
return;
}
- final long ramBefore = reserveDense1D(firstDocID, size);
- final byte[] dense = pointsDenseBuffer();
+ final long ramBefore = reserveDenseRange(firstDocID, size);
+ final byte[] buffer = sharedScratch.bytesScratch();
int remaining = size;
while (remaining > 0) {
int chunk = Math.min(POINTS_BUFFER_LONG_VALUES, remaining);
- cursor.fillLongPoints(dense, 0, chunk);
- bytesOut.writeBytes(dense, 0, chunk * Long.BYTES);
+ cursor.fillLongPoints(buffer, 0, chunk);
+ bytesOut.writeBytes(buffer, 0, chunk * Long.BYTES);
remaining -= chunk;
}
- commitDense1D(firstDocID, size, ramBefore);
+ commitDenseRange(firstDocID, size, ramBefore);
}
- private byte[] pointsDenseBuffer() {
- if (densePointsBuffer == null) {
- densePointsBuffer = new byte[POINTS_BUFFER_BYTES];
+ void addDenseNDValues(int firstDocID, BytesRefValuesCursor cursor) throws IOException {
+ final int size = cursor.size();
+ if (size == 0) {
+ return;
+ }
+ final long ramBefore = reserveDenseRange(firstDocID, size);
+ final int width = packedBytesLength;
+ final int perChunk = SharedIndexingScratch.BYTES_SCRATCH_SIZE / width;
+ final byte[] buffer = sharedScratch.bytesScratch();
+ int remaining = size;
+ while (remaining > 0) {
+ int chunk = Math.min(perChunk, remaining);
+ cursor.fillPackedPoints(buffer, 0, chunk, width);
+ bytesOut.writeBytes(buffer, 0, chunk * width);
+ remaining -= chunk;
}
- return densePointsBuffer;
+ commitDenseRange(firstDocID, size, ramBefore);
}
private void validate1DPacked(int byteWidth) {
@@ -149,7 +170,7 @@ class PointValuesWriter {
}
}
- private long reserveDense1D(int firstDocID, int size) {
+ private long reserveDenseRange(int firstDocID, int size) {
assert firstDocID > lastDocID
: "firstDocID=" + firstDocID + " must be > lastDocID=" + lastDocID;
final int oldLength = docIDs.length;
@@ -163,7 +184,7 @@ class PointValuesWriter {
return bytes.ramBytesUsed();
}
- private void commitDense1D(int firstDocID, int size, long ramBefore) {
+ private void commitDenseRange(int firstDocID, int size, long ramBefore) {
iwBytesUsed.addAndGet(bytes.ramBytesUsed() - ramBefore);
numDocs += size;
lastDocID = firstDocID + size - 1;
diff --git a/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java b/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java
new file mode 100644
index 00000000000..542a2acee0b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import org.apache.lucene.util.Counter;
+
+/**
+ * A holder of lazily-allocated, shared scratch buffers owned by {@link IndexingChain} and provided
+ * to per-field writers that need transient buffers during indexing.
+ *
+ * <p>Because {@link IndexingChain} (and the {@link DocumentsWriterPerThread} it belongs to) indexes
+ * documents single-threadedly, these buffers are safe to reuse across all writers within the same
+ * chain. Callers must treat each buffer as transient scratch: fill it, drain it within the same
+ * call, and not retain a reference across calls.
+ */
+final class SharedIndexingScratch {
+
+ /** Size in bytes of the shared byte scratch buffer. */
+ static final int BYTES_SCRATCH_SIZE = 4 * 1024;
+
+ private final Counter bytesUsed;
+ private byte[] bytesScratchBuffer;
+
+ SharedIndexingScratch(Counter bytesUsed) {
+ this.bytesUsed = bytesUsed;
+ }
+
+ /**
+ * Returns the shared byte scratch buffer, allocating it on the first call and tracking its RAM
+ * via the {@link Counter} supplied at construction.
+ *
+ * <p>Callers must treat the returned array as transient scratch.
+ */
+ byte[] bytesScratch() {
+ if (bytesScratchBuffer == null) {
+ bytesScratchBuffer = new byte[BYTES_SCRATCH_SIZE];
+ bytesUsed.addAndGet(BYTES_SCRATCH_SIZE);
+ }
+ return bytesScratchBuffer;
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java b/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java
index e496603e90f..ab3fa496a1f 100644
--- a/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java
@@ -147,6 +147,125 @@ public class ColumnBatchTestUtil {
}
}
+ /** Dense {@link BinaryColumn} backed by a fixed {@link BytesRef} array, one value per doc. */
+ public static class ArrayDenseBinaryColumn extends BinaryColumn {
+ private final BytesRef[] values;
+
+ public ArrayDenseBinaryColumn(String name, IndexableFieldType fieldType, BytesRef[] values) {
+ super(name, fieldType, Density.DENSE);
+ this.values = values;
+ }
+
+ @Override
+ public ObjectTupleCursor<BytesRef> tuples() {
+ return new ObjectTupleCursor<>() {
+ int pos = -1;
+
+ @Override
+ public int nextDoc() {
+ pos++;
+ return pos < values.length ? pos : DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public BytesRef value() {
+ return values[pos];
+ }
+ };
+ }
+
+ @Override
+ public BytesRefValuesCursor values() {
+ return new BytesRefValuesCursor(values.length) {
+ int pos = 0;
+
+ @Override
+ public BytesRef nextValue() {
+ if (pos >= values.length) {
+ throw new IllegalStateException(
+ "BytesRefValuesCursor exhausted: size=" + values.length);
+ }
+ return values[pos++];
+ }
+ };
+ }
+ }
+
+ /**
+ * Dense {@link BinaryColumn} backed by a single contiguous packed {@code byte[]} of {@code n *
+ * width} bytes.
+ */
+ public static class ContiguousDenseBinaryColumn extends BinaryColumn {
+ private final byte[] packed;
+ private final int width;
+ private final int n;
+
+ /**
+ * @param packed flat array of {@code n * width} bytes; {@code packed[i*width..(i+1)*width)} is
+ * the value for batch-local doc-id {@code i}.
+ * @param width number of bytes per value
+ */
+ public ContiguousDenseBinaryColumn(
+ String name, IndexableFieldType fieldType, byte[] packed, int width) {
+ super(name, fieldType, Density.DENSE);
+ assert packed.length % width == 0 : "packed.length must be a multiple of width";
+ this.packed = packed;
+ this.width = width;
+ this.n = packed.length / width;
+ }
+
+ @Override
+ public ObjectTupleCursor<BytesRef> tuples() {
+ return new ObjectTupleCursor<>() {
+ int pos = -1;
+ final BytesRef ref = new BytesRef(packed, 0, width);
+
+ @Override
+ public int nextDoc() {
+ pos++;
+ if (pos < n) {
+ ref.offset = pos * width;
+ return pos;
+ }
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public BytesRef value() {
+ return ref;
+ }
+ };
+ }
+
+ @Override
+ public BytesRefValuesCursor values() {
+ return new BytesRefValuesCursor(n) {
+ int consumed = 0;
+ final BytesRef ref = new BytesRef(packed, 0, width);
+
+ @Override
+ public BytesRef nextValue() {
+ if (consumed >= n) {
+ throw new IllegalStateException("BytesRefValuesCursor exhausted: size=" + n);
+ }
+ ref.offset = consumed * width;
+ consumed++;
+ return ref;
+ }
+
+ /** Single bulk copy — exercises the override path. */
+ @Override
+ public void fillPackedPoints(byte[] dst, int offset, int length, int width) {
+ if (consumed + length > n) {
+ throw new IllegalStateException("BytesRefValuesCursor exhausted: size=" + n);
+ }
+ System.arraycopy(packed, consumed * width, dst, offset, length * width);
+ consumed += length;
+ }
+ };
+ }
+ }
+
/** Dense {@link LongColumn} with an optional bulk values cursor. */
public static class ArrayDenseLongColumn extends LongColumn {
private final long[] values;
diff --git a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchBinaryColumn.java b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchBinaryColumn.java
index 66579f427bb..de51ecf6127 100644
--- a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchBinaryColumn.java
+++ b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchBinaryColumn.java
@@ -17,11 +17,14 @@
package org.apache.lucene.document.column;
import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayBinaryColumn;
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayDenseBinaryColumn;
import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayLongColumn;
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ContiguousDenseBinaryColumn;
import static org.apache.lucene.document.column.ColumnBatchTestUtil.simpleBatch;
import java.io.IOException;
import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
@@ -50,6 +53,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
/** Tests for {@link BinaryColumn} batch indexing. */
public class TestColumnBatchBinaryColumn extends LuceneTestCase {
@@ -693,4 +697,208 @@ public class TestColumnBatchBinaryColumn extends LuceneTestCase {
w.close();
dir.close();
}
+
+ public void testDenseBinaryPoints() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+
+ // 1-D, 4-byte binary points (sort-encoded ints).
+ FieldType type = new FieldType();
+ type.setDimensions(1, Integer.BYTES);
+ type.freeze();
+
+ int[] rawValues = {-100, 0, 1, 42, 999};
+ BytesRef[] encoded = new BytesRef[rawValues.length];
+ for (int i = 0; i < rawValues.length; i++) {
+ byte[] b = new byte[Integer.BYTES];
+ NumericUtils.intToSortableBytes(rawValues[i], b, 0);
+ encoded[i] = new BytesRef(b);
+ }
+
+ w.addBatch(simpleBatch(rawValues.length, new ArrayDenseBinaryColumn("p", type, encoded)));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ IndexSearcher searcher = new IndexSearcher(r);
+
+ // All docs match a full-range query.
+ byte[] minB = new byte[Integer.BYTES];
+ byte[] maxB = new byte[Integer.BYTES];
+ NumericUtils.intToSortableBytes(Integer.MIN_VALUE, minB, 0);
+ NumericUtils.intToSortableBytes(Integer.MAX_VALUE, maxB, 0);
+ assertEquals(rawValues.length, searcher.count(BinaryPoint.newRangeQuery("p", minB, maxB)));
+
+ // Exact queries for first, last, and a middle value.
+ for (int raw : new int[] {rawValues[0], rawValues[rawValues.length - 1], rawValues[2]}) {
+ byte[] exact = new byte[Integer.BYTES];
+ NumericUtils.intToSortableBytes(raw, exact, 0);
+ assertEquals(1, searcher.count(BinaryPoint.newExactQuery("p", exact)));
+ }
+
+ // A value that was not indexed should match nothing.
+ byte[] absent = new byte[Integer.BYTES];
+ NumericUtils.intToSortableBytes(-1, absent, 0);
+ assertEquals(0, searcher.count(BinaryPoint.newExactQuery("p", absent)));
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testDenseBinaryPointsWithSortedDocValues() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+
+ // SORTED DV + 1-D 4-byte points. Same BytesRef serves both.
+ FieldType type = new FieldType();
+ type.setDocValuesType(org.apache.lucene.index.DocValuesType.SORTED);
+ type.setDimensions(1, Integer.BYTES);
+ type.freeze();
+
+ // 4 docs; sort-encoded ints are used for both DV and points.
+ final int n = 4;
+ BytesRef[] encoded = new BytesRef[n];
+ for (int i = 0; i < n; i++) {
+ byte[] b = new byte[Integer.BYTES];
+ NumericUtils.intToSortableBytes(i * 10, b, 0);
+ encoded[i] = new BytesRef(b);
+ }
+
+ w.addBatch(simpleBatch(n, new ArrayDenseBinaryColumn("sdv", type, encoded)));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher searcher = new IndexSearcher(leaf);
+
+ // Verify sorted DV: all 4 docs should have DV (4 distinct values).
+ SortedDocValues dv = leaf.getSortedDocValues("sdv");
+ assertNotNull(dv);
+ assertEquals(n, dv.getValueCount());
+ for (int i = 0; i < n; i++) {
+ assertEquals(i, dv.nextDoc());
+ assertEquals(encoded[i], dv.lookupOrd(dv.ordValue()));
+ }
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, dv.nextDoc());
+
+ // Verify points: exact query for each encoded value returns exactly 1 doc.
+ for (int i = 0; i < n; i++) {
+ assertEquals(1, searcher.count(BinaryPoint.newExactQuery("sdv", encoded[i].bytes.clone())));
+ }
+ // Range query spanning all values.
+ byte[] minB = new byte[Integer.BYTES];
+ byte[] maxB = new byte[Integer.BYTES];
+ NumericUtils.intToSortableBytes(Integer.MIN_VALUE, minB, 0);
+ NumericUtils.intToSortableBytes(Integer.MAX_VALUE, maxB, 0);
+ assertEquals(n, searcher.count(BinaryPoint.newRangeQuery("sdv", minB, maxB)));
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testDenseBinaryPointsLargeBatch() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+
+ // 1-D, 8-bytes-per-dim binary points: packedLen = 8, chunk = 4096/8 = 512.
+ final int bytesPerDim = Long.BYTES;
+ FieldType type = new FieldType();
+ type.setDimensions(1, bytesPerDim);
+ type.freeze();
+
+ // n = 1100 > 2 * 512 → loop runs at least 3 times.
+ final int n = 1100;
+ BytesRef[] encoded = new BytesRef[n];
+ for (int i = 0; i < n; i++) {
+ byte[] b = new byte[bytesPerDim];
+ NumericUtils.longToSortableBytes(i * 1_000_000L - 100L, b, 0);
+ encoded[i] = new BytesRef(b);
+ }
+
+ w.addBatch(simpleBatch(n, new ArrayDenseBinaryColumn("p1d8", type, encoded)));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ IndexSearcher searcher = new IndexSearcher(r);
+
+ // All docs indexed.
+ byte[] minB = new byte[bytesPerDim];
+ byte[] maxB = new byte[bytesPerDim];
+ NumericUtils.longToSortableBytes(Long.MIN_VALUE, minB, 0);
+ NumericUtils.longToSortableBytes(Long.MAX_VALUE, maxB, 0);
+ assertEquals(n, searcher.count(BinaryPoint.newRangeQuery("p1d8", minB, maxB)));
+
+ // Values at and around the chunk boundary (index 511 / 512) round-trip correctly.
+ assertEquals(1, searcher.count(BinaryPoint.newExactQuery("p1d8", encoded[511].bytes.clone())));
+ assertEquals(1, searcher.count(BinaryPoint.newExactQuery("p1d8", encoded[512].bytes.clone())));
+
+ // A value that was not indexed.
+ byte[] absent = new byte[bytesPerDim];
+ absent[0] = (byte) 0xFF; // sorts above everything we indexed
+ assertEquals(0, searcher.count(BinaryPoint.newExactQuery("p1d8", absent)));
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testDenseBinaryPointsContiguousFill() throws IOException {
+ final int bytesPerDim = Long.BYTES; // packedLen = 8, perChunk = 512
+ final int n = 600; // spans two chunk iterations (512 + 88)
+
+ FieldType type = new FieldType();
+ type.setDimensions(1, bytesPerDim);
+ type.freeze();
+
+ // Build both representations from the same encoded values.
+ BytesRef[] perValueArray = new BytesRef[n];
+ byte[] packedFlat = new byte[n * bytesPerDim];
+ for (int i = 0; i < n; i++) {
+ byte[] b = new byte[bytesPerDim];
+ NumericUtils.longToSortableBytes(i * 500L, b, 0);
+ perValueArray[i] = new BytesRef(b);
+ System.arraycopy(b, 0, packedFlat, i * bytesPerDim, bytesPerDim);
+ }
+
+ // Index via the default per-value path (ArrayDenseBinaryColumn).
+ Directory dirDefault = newDirectory();
+ IndexWriter wDefault = new IndexWriter(dirDefault, newIndexWriterConfig());
+ wDefault.addBatch(simpleBatch(n, new ArrayDenseBinaryColumn("p", type, perValueArray)));
+ DirectoryReader rDefault = DirectoryReader.open(wDefault);
+ IndexSearcher searcherDefault = new IndexSearcher(rDefault);
+
+ // Index via the contiguous bulk-copy override path (ContiguousDenseBinaryColumn).
+ Directory dirBulk = newDirectory();
+ IndexWriter wBulk = new IndexWriter(dirBulk, newIndexWriterConfig());
+ wBulk.addBatch(
+ simpleBatch(n, new ContiguousDenseBinaryColumn("p", type, packedFlat, bytesPerDim)));
+ DirectoryReader rBulk = DirectoryReader.open(wBulk);
+ IndexSearcher searcherBulk = new IndexSearcher(rBulk);
+
+ // Both indexes must return the same results for a full range query.
+ byte[] minB = new byte[bytesPerDim];
+ byte[] maxB = new byte[bytesPerDim];
+ NumericUtils.longToSortableBytes(Long.MIN_VALUE, minB, 0);
+ NumericUtils.longToSortableBytes(Long.MAX_VALUE, maxB, 0);
+ assertEquals(n, searcherDefault.count(BinaryPoint.newRangeQuery("p", minB, maxB)));
+ assertEquals(n, searcherBulk.count(BinaryPoint.newRangeQuery("p", minB, maxB)));
+
+ // Exact queries for a sample of values, including the chunk boundary (index 511/512).
+ for (int idx : new int[] {0, 1, 255, 511, 512, 513, n - 1}) {
+ byte[] exact = perValueArray[idx].bytes.clone();
+ assertEquals(
+ "default path mismatch at idx=" + idx,
+ 1,
+ searcherDefault.count(BinaryPoint.newExactQuery("p", exact)));
+ assertEquals(
+ "bulk override path mismatch at idx=" + idx,
+ 1,
+ searcherBulk.count(BinaryPoint.newExactQuery("p", exact)));
+ }
+
+ rDefault.close();
+ wDefault.close();
+ dirDefault.close();
+ rBulk.close();
+ wBulk.close();
+ dirBulk.close();
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchLongColumn.java b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchLongColumn.java
index b309a016dfb..fe0664a84ad 100644
--- a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchLongColumn.java
+++ b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchLongColumn.java
@@ -1178,7 +1178,7 @@ public class TestColumnBatchLongColumn extends LuceneTestCase {
}
/**
- * Dense long batch larger than {@code POINTS_BUFFER_LONG_VALUES} (128) so the chunked encoding
+ * Dense long batch larger than {@code POINTS_BUFFER_LONG_VALUES} (512) so the chunked encoding
* loop in {@code PointValuesWriter#addDense1DLongValues} runs more than once.
*/
public void testDensePointsLongLargeBatch() throws IOException {
@@ -1189,7 +1189,7 @@ public class TestColumnBatchLongColumn extends LuceneTestCase {
type.setDimensions(1, Long.BYTES);
type.freeze();
- final int n = 350; // > 2 * 128 chunks
+ final int n = 1100; // > 2 * 512 chunks
long[] values = new long[n];
for (int i = 0; i < n; i++) {
values[i] = i * 1_000_000L - 100L; // mix of negative and positive
@@ -1201,8 +1201,8 @@ public class TestColumnBatchLongColumn extends LuceneTestCase {
assertEquals(n, searcher.count(LongPoint.newRangeQuery("v", Long.MIN_VALUE, Long.MAX_VALUE)));
assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[0])));
assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[n - 1])));
- assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[127]))); // chunk boundary
- assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[128])));
+ assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[511]))); // chunk boundary
+ assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[512])));
assertEquals(0, searcher.count(LongPoint.newExactQuery("v", -1L)));
assertEquals(n, searcher.count(LongPoint.newRangeQuery("v", values[0], values[n - 1])));