You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ch...@apache.org on 2026/05/27 11:10:13 UTC
(lucene) branch main updated: Stabilize column batch tests under randomized IWC (#16130)
This is an automated email from the ASF dual-hosted git repository.
ChrisHegarty pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 710e296cdae Stabilize column batch tests under randomized IWC (#16130)
710e296cdae is described below
commit 710e296cdae05e9c28bacff510cc58afedec0ecc
Author: Tim Brooks <ti...@timbrooks.org>
AuthorDate: Wed May 27 05:10:05 2026 -0600
Stabilize column batch tests under randomized IWC (#16130)
Pin TieredMergePolicy and disable auto-flush in tests whose
assertions depend on doc-id ordering across multiple addBatch /
addDocument calls. Without this, randomized small buffers fragment
segments and MockRandomMergePolicy's reorder/TMP's size-sorted forced
merge can swap doc-ids, producing rare flakes at high repeat counts.
testVectorAcrossMultipleBatches switches to a sort-based comparison
since its invariant is value presence, not doc-id position.
---
.../column/TestColumnBatchDictionaryColumn.java} | 30 +++++--
.../lucene/index/TestColumnBatchIndexing.java | 98 ++++++++++++++++------
2 files changed, 96 insertions(+), 32 deletions(-)
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDictionaryColumn.java b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchDictionaryColumn.java
similarity index 96%
rename from lucene/core/src/test/org/apache/lucene/index/TestDictionaryColumn.java
rename to lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchDictionaryColumn.java
index 72f6764f97d..ec9776971eb 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDictionaryColumn.java
+++ b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchDictionaryColumn.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.index;
+package org.apache.lucene.document.column;
import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayDenseDictionaryColumn;
import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayDenseLongColumn;
@@ -35,6 +35,19 @@ import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
@@ -45,15 +58,11 @@ import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
/** Tests for {@link org.apache.lucene.document.column.DictionaryColumn} end-to-end indexing. */
-public class TestDictionaryColumn extends LuceneTestCase {
+public class TestColumnBatchDictionaryColumn extends LuceneTestCase {
private static final List<BytesRef> COLORS =
List.of(new BytesRef("blue"), new BytesRef("green"), new BytesRef("red"));
- // -------------------------------------------------------------------------
- // SORTED doc values — sparse path
- // -------------------------------------------------------------------------
-
public void testSortedSparse() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
@@ -339,8 +348,13 @@ public class TestDictionaryColumn extends LuceneTestCase {
public void testMixedWithPlainDocument() throws IOException {
Directory dir = newDirectory();
- IndexWriter w =
- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(new TieredMergePolicy()));
+ // Disable auto-flush so the addBatch and addDocument share one segment; otherwise a
+ // randomized tiny maxBufferedDocs can split them and TMP's size-sorted forceMerge
+ // may interleave them.
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter w = new IndexWriter(dir, iwc);
// Batch adds doc 0 (blue) and doc 1 (red)
w.addBatch(
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestColumnBatchIndexing.java b/lucene/core/src/test/org/apache/lucene/index/TestColumnBatchIndexing.java
index d505b721495..e531ff5f5f5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestColumnBatchIndexing.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestColumnBatchIndexing.java
@@ -30,6 +30,8 @@ import static org.apache.lucene.document.column.ColumnBatchTestUtil.simpleBatch;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
@@ -444,7 +446,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
// --- Batch path ---
Directory batchDir = newDirectory();
- try (IndexWriter batchW = new IndexWriter(batchDir, newIndexWriterConfig())) {
+ IndexWriterConfig batchIwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ batchIwc.setRAMBufferSizeMB(16);
+ batchIwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ try (IndexWriter batchW = new IndexWriter(batchDir, batchIwc)) {
BytesRef[] refs = new BytesRef[values.length];
for (int i = 0; i < values.length; i++) {
refs[i] = newBytesRef(values[i]);
@@ -459,9 +464,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
}
Directory singleDir = newDirectory();
- try (IndexWriter singleW =
- new IndexWriter(
- singleDir, newIndexWriterConfig().setMergePolicy(new TieredMergePolicy()))) {
+ IndexWriterConfig singleIwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ singleIwc.setRAMBufferSizeMB(16);
+ singleIwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ try (IndexWriter singleW = new IndexWriter(singleDir, singleIwc)) {
int next = 0;
for (int d = 0; d < totalDocs; d++) {
Document doc = new Document();
@@ -592,12 +598,22 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
FloatVectorValues values = leaf.getFloatVectorValues("v");
KnnVectorValues.DocIndexIterator it = values.iterator();
- float[][] all = {firstBatch[0], firstBatch[1], secondBatch[0], secondBatch[1], secondBatch[2]};
- for (int i = 0; i < all.length; i++) {
- assertEquals(i, it.nextDoc());
- assertArrayEquals(all[i], values.vectorValue(it.index()), 0f);
+ List<float[]> collected = new ArrayList<>();
+ while (it.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ collected.add(values.vectorValue(it.index()).clone());
+ }
+ // Merge order is not guaranteed when batches flush to separate segments,
+ // so compare as a sorted set rather than by doc-ID position.
+ Comparator<float[]> byFirst = Comparator.comparingDouble(v -> v[0]);
+ collected.sort(byFirst);
+ float[][] expected = {
+ firstBatch[0], firstBatch[1], secondBatch[0], secondBatch[1], secondBatch[2]
+ };
+ Arrays.sort(expected, byFirst);
+ assertEquals(expected.length, collected.size());
+ for (int i = 0; i < expected.length; i++) {
+ assertArrayEquals(expected[i], collected.get(i), 0f);
}
- assertEquals(DocIdSetIterator.NO_MORE_DOCS, it.nextDoc());
r.close();
w.close();
@@ -663,8 +679,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
public void testLongTupleCursorThrowMidBatchRecovers() throws IOException {
Directory dir = newDirectory();
- IndexWriter w =
- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(new TieredMergePolicy()));
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter w = new IndexWriter(dir, iwc);
// 5-doc batch; LongTupleCursor#nextDoc throws at index 2 (during column pass).
int[] docIds = {0, 1, 2, 3, 4};
@@ -719,8 +737,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
public void testLongValuesCursorThrowMidBatchRecovers() throws IOException {
Directory dir = newDirectory();
- IndexWriter w =
- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(new TieredMergePolicy()));
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter w = new IndexWriter(dir, iwc);
// 6-doc dense batch; LongValuesCursor#fillDocValues throws partway (after 3 fills).
long[] values = {1, 2, 3, 4, 5, 6};
@@ -771,7 +791,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
public void testBinaryTupleCursorThrowMidBatchRecovers() throws IOException {
Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter w = new IndexWriter(dir, iwc);
int[] docIds = {0, 1, 2, 3};
BytesRef[] values = {
@@ -815,7 +838,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
public void testVectorTupleCursorThrowMidBatchRecovers() throws IOException {
Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter w = new IndexWriter(dir, iwc);
FieldType vectorType = floatVectorType(2, VectorSimilarityFunction.EUCLIDEAN);
int[] docIds = {0, 1, 2};
@@ -882,8 +908,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
public void testMultipleBatchesIntoOneSegment() throws IOException {
Directory dir = newDirectory();
- IndexWriter w =
- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(new TieredMergePolicy()));
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter w = new IndexWriter(dir, iwc);
w.addBatch(
simpleBatch(
@@ -919,8 +947,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
*/
public void testStoredFieldsAppearAfterStoredFreeBatch() throws IOException {
Directory dir = newDirectory();
- IndexWriter w =
- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(new TieredMergePolicy()));
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter w = new IndexWriter(dir, iwc);
// Batch A: 3 docs, indexed-only string column (no stored).
BytesRef[] ids = {newBytesRef("a"), newBytesRef("b"), newBytesRef("c")};
@@ -1005,8 +1035,10 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
/** Interleaving addBatch and addDocument keeps doc-ids contiguous and values consistent. */
public void testAddBatchInterleavedWithAddDocument() throws IOException {
Directory dir = newDirectory();
- IndexWriter w =
- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(new TieredMergePolicy()));
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter w = new IndexWriter(dir, iwc);
// batch (3) → doc → batch (2) → doc, expecting docs 0..6 in order.
w.addBatch(
@@ -1104,7 +1136,12 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
Directory dir = newDirectory();
// Pin a non-mock merge policy: MockRandomMergePolicy.reorder may reverse doc-ids during
// merge, which is legitimate behavior but defeats the strict per-doc-id assertions below.
+ // Also disable auto-flush so the 3 addDocument calls land in a single segment; otherwise a
+ // randomized tiny maxBufferedDocs splits them into multiple small segments and TMP's
+ // size-sorted forceMerge can interleave them with the addBatch segment.
IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter w = new IndexWriter(dir, iwc);
// Segment A via addDocument: values 10, 20, 30.
@@ -1243,7 +1280,11 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
}
private static void buildDocByDocIndex(Directory dir, ParityDoc[] docs) throws IOException {
- try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())))) {
+ IndexWriterConfig iwc =
+ new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ try (IndexWriter w = new IndexWriter(dir, iwc)) {
for (ParityDoc d : docs) {
Document doc = new Document();
if (d.num != null) doc.add(new NumericDocValuesField("num", d.num));
@@ -1295,7 +1336,11 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
vecType.setVectorAttributes(2, VectorEncoding.FLOAT32, VectorSimilarityFunction.EUCLIDEAN);
vecType.freeze();
- try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())))) {
+ IndexWriterConfig iwc =
+ new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ try (IndexWriter w = new IndexWriter(dir, iwc)) {
for (int bi = 0; bi < numBatches; bi++) {
int from = starts[bi];
int toExcl = starts[bi + 1];
@@ -1698,7 +1743,12 @@ public class TestColumnBatchIndexing extends LuceneTestCase {
public void testSameFieldNameAcrossBatchesAllowed() throws IOException {
Directory dir = newDirectory();
- try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
+ // Pin a stable IWC so randomized small buffers / merge policies cannot reorder
+ // this test asserts doc-id ordering.
+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(new TieredMergePolicy());
+ iwc.setRAMBufferSizeMB(16);
+ iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ try (IndexWriter w = new IndexWriter(dir, iwc)) {
w.addBatch(
simpleBatch(
1,