(lucene) branch branch_10x updated: Randomize maxDocValuesOverlays in tests (GH#16644) (#16645)

[ji...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jimczi pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_10x by this push:
     new 08f3367332c Randomize maxDocValuesOverlays in tests (GH#16644) (#16645)
08f3367332c is described below

commit 08f3367332c3c5679749f32c1ff5583a829ce0be
Author: Jim Ferenczi <ji...@apache.org>
AuthorDate: Mon Sep 7 12:09:26 2026 +0200

    Randomize maxDocValuesOverlays in tests (GH#16644) (#16645)
    
    Backport of #16645. Randomize the overlay budget in newIndexWriterConfig
    so update-heavy tests stay within the handle limit and the lower budgets
    (including the disabled path at 0) get exercised. The dedicated stacking
    and folding test uses an explicit budget here since the default is 0 on
    10.x.
---
 .../index/TestIncrementalDocValuesUpdates.java     | 54 ++++++++++++++++++++++
 .../apache/lucene/tests/util/LuceneTestCase.java   | 11 +++++
 2 files changed, 65 insertions(+)

diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java
index de5fa719dc6..4d439112ef9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java
@@ -50,6 +50,60 @@ public class TestIncrementalDocValuesUpdates extends LuceneTestCase {
     assertNumericField(reader, id, "val", expected);
   }
 
+  /**
+   * Exercises a high overlay budget under heavy stacking and repeated folding, which neither {@link
+   * #newIndexWriterConfig} (small randomized budget) nor {@link #incrementalConfig} (low range)
+   * reaches. Every update is resolved to its own generation via a reader reopen so overlays stack
+   * up to the budget and fold, across two independent numeric fields, and every value must still
+   * read back correctly. Occasional force-merges keep the segment (and open file handle) count
+   * bounded so it also runs under the default open-handle limit.
+   */
+  public void testHighBudgetHeavyStackingAndFolding() throws Exception {
+    IndexWriterConfig conf =
+        new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(16);
+    try (Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, conf)) {
+      final int numDocs = atLeast(50);
+      Map<String, Long> expectedA = new HashMap<>();
+      Map<String, Long> expectedB = new HashMap<>();
+      for (int i = 0; i < numDocs; i++) {
+        Document d = new Document();
+        d.add(new StringField("id", "d" + i, StringField.Store.NO));
+        d.add(new NumericDocValuesField("a", i));
+        d.add(new NumericDocValuesField("b", -i));
+        expectedA.put("d" + i, (long) i);
+        expectedB.put("d" + i, (long) -i);
+        w.addDocument(d);
+      }
+      w.commit();
+      final int updates = atLeast(300);
+      for (int i = 0; i < updates; i++) {
+        String id = "d" + random().nextInt(numDocs);
+        long v = random().nextLong();
+        String field = random().nextBoolean() ? "a" : "b";
+        w.updateNumericDocValue(new Term("id", id), field, v);
+        (field.equals("a") ? expectedA : expectedB).put(id, v);
+        // Resolve each update into its own generation so overlays actually stack (and fold at the
+        // budget) rather than collapsing into a single batched write.
+        try (DirectoryReader r = DirectoryReader.open(w)) {
+          assertNotNull(r);
+        }
+        if (rarely()) {
+          // Keep the segment count (hence the open-handle count) bounded.
+          w.forceMerge(1 + random().nextInt(2));
+        }
+      }
+      try (DirectoryReader reader = DirectoryReader.open(w)) {
+        for (Map.Entry<String, Long> e : expectedA.entrySet()) {
+          assertNumericField(reader, e.getKey(), "a", e.getValue());
+        }
+        for (Map.Entry<String, Long> e : expectedB.entrySet()) {
+          assertNumericField(reader, e.getKey(), "b", e.getValue());
+        }
+      }
+    }
+  }
+
   private static void assertNumericField(IndexReader reader, String id, String field, long expected)
       throws IOException {
     for (LeafReaderContext ctx : reader.leaves()) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
index d0b057d267a..15acaf7ebfe 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
@@ -1049,6 +1049,17 @@ public abstract class LuceneTestCase extends Assert {
     }
 
     c.setMaxFullFlushMergeWaitMillis(rarely() ? atLeast(r, 1000) : atLeast(r, 200));
+
+    // Randomize the incremental doc-values overlay budget (GH#16418) over a small, bounded range.
+    // A field's reader keeps one producer open per live generation (base plus up to this many
+    // sparse deltas), so open file handles scale with segments x updated-fields x overlays; a fixed
+    // budget would both push update-heavy tests past the open-handle limit and leave the lower
+    // budgets (including the disabled path at 0) uncovered. Tests that need a specific budget set
+    // it
+    // explicitly after newIndexWriterConfig and, if update-heavy, annotate a matching
+    // @HandleLimitFS.MaxOpenHandles.
+    c.setMaxDocValuesOverlays(TestUtil.nextInt(r, 0, 4));
+
     return c;
   }