You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2025/03/16 14:54:48 UTC
(lucene) branch branch_10x updated: Decode doc ids in BKD leaves with auto-vectorized loops (#14203)
This is an automated email from the ASF dual-hosted git repository.
guofeng pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 53cef55e562 Decode doc ids in BKD leaves with auto-vectorized loops (#14203)
53cef55e562 is described below
commit 53cef55e562307e5f946f6ee1f0747e15e753ee6
Author: gf2121 <52...@users.noreply.github.com>
AuthorDate: Sun Mar 16 22:20:07 2025 +0800
Decode doc ids in BKD leaves with auto-vectorized loops (#14203)
---
lucene/CHANGES.txt | 2 +
.../codecs/lucene90/Lucene90PointsFormat.java | 30 +++-
.../codecs/lucene90/Lucene90PointsWriter.java | 28 +++-
.../java/org/apache/lucene/util/bkd/BKDReader.java | 10 +-
.../java/org/apache/lucene/util/bkd/BKDWriter.java | 31 +++-
.../org/apache/lucene/util/bkd/DocIdsWriter.java | 175 ++++++++++++++++-----
.../lucene90/TestLucene90PointsFormatV0.java | 35 +++++
.../apache/lucene/util/bkd/TestDocIdsWriter.java | 22 ++-
8 files changed, 272 insertions(+), 61 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ae191395250..fe5286aac80 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -132,6 +132,8 @@ Optimizations
* GITHUB#14301: Avoid unnecessary evaluations and skipping documents. (hanbj)
+# GITHUB#14203: Decode doc ids in BKD leaves with auto-vectorized loops when using DEFAULT_MAX_POINTS_IN_LEAF_NODE. (Guo Feng)
+
Bug Fixes
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsFormat.java
index 64e61991eb1..a59b0181463 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsFormat.java
@@ -17,11 +17,13 @@
package org.apache.lucene.codecs.lucene90;
import java.io.IOException;
+import java.util.Map;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.PointsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.bkd.BKDWriter;
/**
* Lucene 9.0 point format, which encodes dimensional values in a block KD-tree structure for fast
@@ -59,18 +61,40 @@ public final class Lucene90PointsFormat extends PointsFormat {
public static final String META_EXTENSION = "kdm";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_BKD_VECTORIZED_BPV24 = 1;
+ static final int VERSION_CURRENT = VERSION_BKD_VECTORIZED_BPV24;
+
+ private static final Map<Integer, Integer> VERSION_TO_BKD_VERSION =
+ Map.of(
+ VERSION_START, BKDWriter.VERSION_META_FILE,
+ VERSION_BKD_VECTORIZED_BPV24, BKDWriter.VERSION_VECTORIZED_DOCID);
+
+ private final int version;
/** Sole constructor */
- public Lucene90PointsFormat() {}
+ public Lucene90PointsFormat() {
+ this(VERSION_CURRENT);
+ }
+
+ /** Constructor that takes a version. This is used for testing with older versions. */
+ Lucene90PointsFormat(int version) {
+ if (VERSION_TO_BKD_VERSION.containsKey(version) == false) {
+ throw new IllegalArgumentException("Invalid version: " + version);
+ }
+ this.version = version;
+ }
@Override
public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new Lucene90PointsWriter(state);
+ return new Lucene90PointsWriter(state, version);
}
@Override
public PointsReader fieldsReader(SegmentReadState state) throws IOException {
return new Lucene90PointsReader(state);
}
+
+ static int bkdVersion(int version) {
+ return VERSION_TO_BKD_VERSION.get(version);
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java
index 2de98e55688..456d12cd962 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java
@@ -46,16 +46,18 @@ public class Lucene90PointsWriter extends PointsWriter {
final SegmentWriteState writeState;
final int maxPointsInLeafNode;
final double maxMBSortInHeap;
+ final int version;
private boolean finished;
/** Full constructor */
public Lucene90PointsWriter(
- SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap)
+ SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap, int version)
throws IOException {
assert writeState.fieldInfos.hasPointValues();
this.writeState = writeState;
this.maxPointsInLeafNode = maxPointsInLeafNode;
this.maxMBSortInHeap = maxMBSortInHeap;
+ this.version = version;
String dataFileName =
IndexFileNames.segmentFileName(
writeState.segmentInfo.name,
@@ -105,6 +107,12 @@ public class Lucene90PointsWriter extends PointsWriter {
}
}
+ public Lucene90PointsWriter(
+ SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap)
+ throws IOException {
+ this(writeState, maxPointsInLeafNode, maxMBSortInHeap, Lucene90PointsFormat.VERSION_CURRENT);
+ }
+
/**
* Uses the defaults values for {@code maxPointsInLeafNode} (512) and {@code maxMBSortInHeap}
* (16.0)
@@ -113,7 +121,17 @@ public class Lucene90PointsWriter extends PointsWriter {
this(
writeState,
BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
- BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP);
+ BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP,
+ Lucene90PointsFormat.VERSION_CURRENT);
+ }
+
+ /** Constructor that takes a version. This is used for testing with older versions. */
+ Lucene90PointsWriter(SegmentWriteState writeState, int version) throws IOException {
+ this(
+ writeState,
+ BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
+ BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP,
+ version);
}
@Override
@@ -135,7 +153,8 @@ public class Lucene90PointsWriter extends PointsWriter {
writeState.segmentInfo.name,
config,
maxMBSortInHeap,
- values.size())) {
+ values.size(),
+ Lucene90PointsFormat.bkdVersion(version))) {
if (values instanceof MutablePointTree) {
IORunnable finalizer =
@@ -233,7 +252,8 @@ public class Lucene90PointsWriter extends PointsWriter {
writeState.segmentInfo.name,
config,
maxMBSortInHeap,
- totMaxSize)) {
+ totMaxSize,
+ Lucene90PointsFormat.bkdVersion(version))) {
List<PointValues> pointValues = new ArrayList<>();
List<MergeState.DocMap> docMaps = new ArrayList<>();
for (int i = 0; i < mergeState.pointsReaders.length; i++) {
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index 17b0cb79845..40f995e61ea 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -34,7 +34,6 @@ import org.apache.lucene.util.MathUtil;
* @lucene.experimental
*/
public class BKDReader extends PointValues {
-
final BKDConfig config;
final int numLeaves;
final IndexInput in;
@@ -269,7 +268,7 @@ public class BKDReader extends PointValues {
1,
minPackedValue,
maxPackedValue,
- new BKDReaderDocIDSetIterator(config.maxPointsInLeafNode()),
+ new BKDReaderDocIDSetIterator(config.maxPointsInLeafNode(), version),
new byte[config.packedBytesLength()],
new byte[config.packedIndexBytesLength()],
new byte[config.packedIndexBytesLength()],
@@ -598,7 +597,8 @@ public class BKDReader extends PointValues {
// How many points are stored in this leaf cell:
int count = leafNodes.readVInt();
// No need to call grow(), it has been called up-front
- docIdsWriter.readInts(leafNodes, count, visitor);
+ // Borrow scratchIterator.docIds as decoding buffer
+ docIdsWriter.readInts(leafNodes, count, visitor, scratchIterator.docIDs);
} else {
pushLeft();
addAll(visitor, grown);
@@ -1036,9 +1036,9 @@ public class BKDReader extends PointValues {
final int[] docIDs;
private final DocIdsWriter docIdsWriter;
- public BKDReaderDocIDSetIterator(int maxPointsInLeafNode) {
+ public BKDReaderDocIDSetIterator(int maxPointsInLeafNode, int version) {
this.docIDs = new int[maxPointsInLeafNode];
- this.docIdsWriter = new DocIdsWriter(maxPointsInLeafNode);
+ this.docIdsWriter = new DocIdsWriter(maxPointsInLeafNode, version);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 2ce7d7e87ef..5db1af560c2 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -85,7 +85,8 @@ public class BKDWriter implements Closeable {
public static final int VERSION_SELECTIVE_INDEXING = 6;
public static final int VERSION_LOW_CARDINALITY_LEAVES = 7;
public static final int VERSION_META_FILE = 9;
- public static final int VERSION_CURRENT = VERSION_META_FILE;
+ public static final int VERSION_VECTORIZED_DOCID = 10;
+ public static final int VERSION_CURRENT = VERSION_VECTORIZED_DOCID;
/** Number of splits before we compute the exact bounding box of an inner node. */
private static final int SPLITS_BEFORE_EXACT_BOUNDS = 4;
@@ -103,6 +104,7 @@ public class BKDWriter implements Closeable {
final TrackingDirectoryWrapper tempDir;
final String tempFileNamePrefix;
final double maxMBSortInHeap;
+ final int version;
final byte[] scratchDiff;
final byte[] scratch;
@@ -139,6 +141,29 @@ public class BKDWriter implements Closeable {
BKDConfig config,
double maxMBSortInHeap,
long totalPointCount) {
+ this(
+ maxDoc,
+ tempDir,
+ tempFileNamePrefix,
+ config,
+ maxMBSortInHeap,
+ totalPointCount,
+ BKDWriter.VERSION_CURRENT);
+ }
+
+ /** This ctor should be only used for testing with older versions. */
+ public BKDWriter(
+ int maxDoc,
+ Directory tempDir,
+ String tempFileNamePrefix,
+ BKDConfig config,
+ double maxMBSortInHeap,
+ long totalPointCount,
+ int version) {
+ if (version < VERSION_START || version > VERSION_CURRENT) {
+ throw new IllegalArgumentException("Version out of range: " + version);
+ }
+ this.version = version;
verifyParams(maxMBSortInHeap, totalPointCount);
// We use tracking dir to deal with removing files on exception, so each place that
// creates temp files doesn't need crazy try/finally/sucess logic:
@@ -165,7 +190,7 @@ public class BKDWriter implements Closeable {
// Maximum number of points we hold in memory at any time
maxPointsSortInHeap = (int) ((maxMBSortInHeap * 1024 * 1024) / (config.bytesPerDoc()));
- docIdsWriter = new DocIdsWriter(config.maxPointsInLeafNode());
+ docIdsWriter = new DocIdsWriter(config.maxPointsInLeafNode(), version);
// Finally, we must be able to hold at least the leaf node in heap during build:
if (maxPointsSortInHeap < config.maxPointsInLeafNode()) {
throw new IllegalArgumentException(
@@ -1245,7 +1270,7 @@ public class BKDWriter implements Closeable {
byte[] packedIndex,
long dataStartFP)
throws IOException {
- CodecUtil.writeHeader(metaOut, CODEC_NAME, VERSION_CURRENT);
+ CodecUtil.writeHeader(metaOut, CODEC_NAME, version);
metaOut.writeVInt(config.numDims());
metaOut.writeVInt(config.numIndexDims());
metaOut.writeVInt(countPerLeaf);
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java
index d188e1881d5..bcb7d964868 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/DocIdsWriter.java
@@ -52,13 +52,16 @@ final class DocIdsWriter {
*/
private final IntsRef scratchIntsRef = new IntsRef();
+ private final int version;
+
{
// This is here to not rely on the default constructor of IntsRef to set offset to 0
scratchIntsRef.offset = 0;
}
- DocIdsWriter(int maxPointsInLeaf) {
- scratch = new int[maxPointsInLeaf];
+ DocIdsWriter(int maxPointsInLeaf, int version) {
+ this.scratch = new int[maxPointsInLeaf];
+ this.version = version;
}
void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws IOException {
@@ -101,7 +104,7 @@ final class DocIdsWriter {
scratch[i] = docIds[start + i] - min;
}
out.writeVInt(min);
- final int halfLen = count >>> 1;
+ final int halfLen = count >> 1;
for (int i = 0; i < halfLen; ++i) {
scratch[i] = scratch[halfLen + i] | (scratch[i] << 16);
}
@@ -114,31 +117,28 @@ final class DocIdsWriter {
} else {
if (max <= 0xFFFFFF) {
out.writeByte(BPV_24);
- // write them the same way we are reading them.
- int i;
- for (i = 0; i < count - 7; i += 8) {
- int doc1 = docIds[start + i];
- int doc2 = docIds[start + i + 1];
- int doc3 = docIds[start + i + 2];
- int doc4 = docIds[start + i + 3];
- int doc5 = docIds[start + i + 4];
- int doc6 = docIds[start + i + 5];
- int doc7 = docIds[start + i + 6];
- int doc8 = docIds[start + i + 7];
- long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 | ((doc3 >>> 8) & 0xffffL);
- long l2 =
- (doc3 & 0xffL) << 56
- | (doc4 & 0xffffffL) << 32
- | (doc5 & 0xffffffL) << 8
- | ((doc6 >> 16) & 0xffL);
- long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8 & 0xffffffL);
- out.writeLong(l1);
- out.writeLong(l2);
- out.writeLong(l3);
- }
- for (; i < count; ++i) {
- out.writeShort((short) (docIds[start + i] >>> 8));
- out.writeByte((byte) docIds[start + i]);
+ if (version < BKDWriter.VERSION_VECTORIZED_DOCID) {
+ writeScalarInts24(docIds, start, count, out);
+ } else {
+ // encode the docs in the format that can be vectorized decoded.
+ final int quarter = count >> 2;
+ final int numInts = quarter * 3;
+ for (int i = 0; i < numInts; i++) {
+ scratch[i] = docIds[i + start] << 8;
+ }
+ for (int i = 0; i < quarter; i++) {
+ final int longIdx = i + numInts + start;
+ scratch[i] |= docIds[longIdx] & 0xFF;
+ scratch[i + quarter] |= (docIds[longIdx] >>> 8) & 0xFF;
+ scratch[i + quarter * 2] |= docIds[longIdx] >>> 16;
+ }
+ for (int i = 0; i < numInts; i++) {
+ out.writeInt(scratch[i]);
+ }
+ for (int i = quarter << 2; i < count; ++i) {
+ out.writeShort((short) docIds[start + i]);
+ out.writeByte((byte) (docIds[start + i] >>> 16));
+ }
}
} else {
out.writeByte(BPV_32);
@@ -149,6 +149,35 @@ final class DocIdsWriter {
}
}
+ private static void writeScalarInts24(int[] docIds, int start, int count, DataOutput out)
+ throws IOException {
+ int i;
+ for (i = 0; i < count - 7; i += 8) {
+ int doc1 = docIds[start + i];
+ int doc2 = docIds[start + i + 1];
+ int doc3 = docIds[start + i + 2];
+ int doc4 = docIds[start + i + 3];
+ int doc5 = docIds[start + i + 4];
+ int doc6 = docIds[start + i + 5];
+ int doc7 = docIds[start + i + 6];
+ int doc8 = docIds[start + i + 7];
+ long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 | ((doc3 >>> 8) & 0xffffL);
+ long l2 =
+ (doc3 & 0xffL) << 56
+ | (doc4 & 0xffffffL) << 32
+ | (doc5 & 0xffffffL) << 8
+ | ((doc6 >> 16) & 0xffL);
+ long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8 & 0xffffffL);
+ out.writeLong(l1);
+ out.writeLong(l2);
+ out.writeLong(l3);
+ }
+ for (; i < count; ++i) {
+ out.writeShort((short) (docIds[start + i] >>> 8));
+ out.writeByte((byte) docIds[start + i]);
+ }
+ }
+
private static void writeIdsAsBitSet(int[] docIds, int start, int count, DataOutput out)
throws IOException {
int min = docIds[start];
@@ -196,7 +225,11 @@ final class DocIdsWriter {
readDelta16(in, count, docIDs);
break;
case BPV_24:
- readInts24(in, count, docIDs);
+ if (version < BKDWriter.VERSION_VECTORIZED_DOCID) {
+ readScalarInts24(in, count, docIDs);
+ } else {
+ readInts24(in, count, docIDs);
+ }
break;
case BPV_32:
readInts32(in, count, docIDs);
@@ -248,21 +281,65 @@ final class DocIdsWriter {
assert pos == count : "pos: " + pos + ", count: " + count;
}
- private static void readDelta16(IndexInput in, int count, int[] docIDs) throws IOException {
+ private static void readDelta16(IndexInput in, int count, int[] docIds) throws IOException {
final int min = in.readVInt();
- final int halfLen = count >>> 1;
- in.readInts(docIDs, 0, halfLen);
- for (int i = 0; i < halfLen; ++i) {
- int l = docIDs[i];
+ final int half = count >> 1;
+ in.readInts(docIds, 0, half);
+ if (count == BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE) {
+ // Same format, but enabling the JVM to specialize the decoding logic for the default number
+ // of points per node proved to help on benchmarks
+ decode16(docIds, BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE / 2, min);
+ } else {
+ decode16(docIds, half, min);
+ }
+ // read the remaining doc if count is odd.
+ for (int i = half << 1; i < count; i++) {
+ docIds[i] = Short.toUnsignedInt(in.readShort()) + min;
+ }
+ }
+
+ private static void decode16(int[] docIDs, int half, int min) {
+ for (int i = 0; i < half; ++i) {
+ final int l = docIDs[i];
docIDs[i] = (l >>> 16) + min;
- docIDs[halfLen + i] = (l & 0xFFFF) + min;
+ docIDs[i + half] = (l & 0xFFFF) + min;
}
- if ((count & 1) == 1) {
- docIDs[count - 1] = Short.toUnsignedInt(in.readShort()) + min;
+ }
+
+ private void readInts24(IndexInput in, int count, int[] docIDs) throws IOException {
+ int quarter = count >> 2;
+ int numInts = quarter * 3;
+ in.readInts(scratch, 0, numInts);
+ if (count == BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE) {
+ // Same format, but enabling the JVM to specialize the decoding logic for the default number
+ // of points per node proved to help on benchmarks
+ decode24(
+ docIDs,
+ scratch,
+ BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE / 4,
+ BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE / 4 * 3);
+ } else {
+ decode24(docIDs, scratch, quarter, numInts);
+ }
+ // Now read the remaining 0, 1, 2 or 3 values
+ for (int i = quarter << 2; i < count; ++i) {
+ docIDs[i] = (in.readShort() & 0xFFFF) | (in.readByte() & 0xFF) << 16;
+ }
+ }
+
+ private static void decode24(int[] docIDs, int[] scratch, int quarter, int numInts) {
+ for (int i = 0; i < numInts; ++i) {
+ docIDs[i] = scratch[i] >>> 8;
+ }
+ for (int i = 0; i < quarter; i++) {
+ docIDs[i + numInts] =
+ (scratch[i] & 0xFF)
+ | ((scratch[i + quarter] & 0xFF) << 8)
+ | ((scratch[i + quarter * 2] & 0xFF) << 16);
}
}
- private static void readInts24(IndexInput in, int count, int[] docIDs) throws IOException {
+ private static void readScalarInts24(IndexInput in, int count, int[] docIDs) throws IOException {
int i;
for (i = 0; i < count - 7; i += 8) {
long l1 = in.readLong();
@@ -290,7 +367,8 @@ final class DocIdsWriter {
* Read {@code count} integers and feed the result directly to {@link
* IntersectVisitor#visit(int)}.
*/
- void readInts(IndexInput in, int count, IntersectVisitor visitor) throws IOException {
+ void readInts(IndexInput in, int count, IntersectVisitor visitor, int[] buffer)
+ throws IOException {
final int bpv = in.readByte();
switch (bpv) {
case CONTINUOUS_IDS:
@@ -303,7 +381,11 @@ final class DocIdsWriter {
readDelta16(in, count, visitor);
break;
case BPV_24:
- readInts24(in, count, visitor);
+ if (version < BKDWriter.VERSION_VECTORIZED_DOCID) {
+ readScalarInts24(in, count, visitor);
+ } else {
+ readInts24(in, count, visitor, buffer);
+ }
break;
case BPV_32:
readInts32(in, count, visitor);
@@ -343,8 +425,17 @@ final class DocIdsWriter {
visitor.visit(scratchIntsRef);
}
- private void readInts24(IndexInput in, int count, IntersectVisitor visitor) throws IOException {
- readInts24(in, count, scratch);
+ private void readInts24(IndexInput in, int count, IntersectVisitor visitor, int[] buffer)
+ throws IOException {
+ readInts24(in, count, buffer);
+ scratchIntsRef.ints = buffer;
+ scratchIntsRef.length = count;
+ visitor.visit(scratchIntsRef);
+ }
+
+ private void readScalarInts24(IndexInput in, int count, IntersectVisitor visitor)
+ throws IOException {
+ readScalarInts24(in, count, scratch);
scratchIntsRef.ints = scratch;
scratchIntsRef.length = count;
visitor.visit(scratchIntsRef);
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PointsFormatV0.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PointsFormatV0.java
new file mode 100644
index 00000000000..fd27eef0488
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PointsFormatV0.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene90;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
+import org.apache.lucene.tests.index.BasePointsFormatTestCase;
+
+public class TestLucene90PointsFormatV0 extends BasePointsFormatTestCase {
+
+ @Override
+ protected Codec getCodec() {
+ return new AssertingCodec() {
+ @Override
+ public PointsFormat pointsFormat() {
+ return new Lucene90PointsFormat(Lucene90PointsFormat.VERSION_START);
+ }
+ };
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java
index 086eb460e56..e0ccea1eea0 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java
@@ -36,11 +36,18 @@ import org.apache.lucene.util.Constants;
public class TestDocIdsWriter extends LuceneTestCase {
+ private static final int[] VERSIONS =
+ new int[] {BKDWriter.VERSION_META_FILE, BKDWriter.VERSION_CURRENT};
+
public void testRandom() throws Exception {
int numIters = atLeast(100);
try (Directory dir = newDirectory()) {
for (int iter = 0; iter < numIters; ++iter) {
- int[] docIDs = new int[1 + random().nextInt(5000)];
+ int count =
+ random().nextBoolean()
+ ? 1 + random().nextInt(5000)
+ : BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE;
+ int[] docIDs = new int[count];
final int bpv = TestUtil.nextInt(random(), 1, 32);
for (int i = 0; i < docIDs.length; ++i) {
docIDs[i] = TestUtil.nextInt(random(), 0, (1 << bpv) - 1);
@@ -69,7 +76,11 @@ public class TestDocIdsWriter extends LuceneTestCase {
int numIters = atLeast(100);
try (Directory dir = newDirectory()) {
for (int iter = 0; iter < numIters; ++iter) {
- int[] docIDs = new int[1 + random().nextInt(5000)];
+ int count =
+ random().nextBoolean()
+ ? 1 + random().nextInt(5000)
+ : BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE;
+ int[] docIDs = new int[count];
int min = random().nextInt(1000);
final int bpv = TestUtil.nextInt(random(), 1, 16);
for (int i = 0; i < docIDs.length; ++i) {
@@ -113,7 +124,9 @@ public class TestDocIdsWriter extends LuceneTestCase {
private void test(Directory dir, int[] ints) throws Exception {
final long len;
- DocIdsWriter docIdsWriter = new DocIdsWriter(ints.length);
+ // It is hard to get BPV24-encoded docs in TextLuceneXXPointsFormat, test bwc here as well.
+ final int version = VERSIONS[random().nextInt(VERSIONS.length)];
+ DocIdsWriter docIdsWriter = new DocIdsWriter(ints.length, version);
try (IndexOutput out = dir.createOutput("tmp", IOContext.DEFAULT)) {
docIdsWriter.writeDocIds(ints, 0, ints.length, out);
len = out.getFilePointer();
@@ -149,7 +162,8 @@ public class TestDocIdsWriter extends LuceneTestCase {
public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
throw new UnsupportedOperationException();
}
- });
+ },
+ new int[ints.length]);
assertArrayEquals(ints, read);
assertEquals(len, in.getFilePointer());
}