You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ji...@apache.org on 2026/08/25 07:02:24 UTC
(lucene) branch main updated: Add tests for incremental doc-values updates (#16418) (#16550)
This is an automated email from the ASF dual-hosted git repository.
jimczi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 2822b28b5ab Add tests for incremental doc-values updates (#16418) (#16550)
2822b28b5ab is described below
commit 2822b28b5ab042672d6cb569589128e5dcafa648
Author: Jim Ferenczi <ji...@apache.org>
AuthorDate: Tue Aug 25 09:02:14 2026 +0200
Add tests for incremental doc-values updates (#16418) (#16550)
Cover the soft-deletes field riding the overlay fold lifecycle, the
addIndexes(Directory) copy-as-is carry-over, and an explicit checkIndex on the deep sparse-fold-over-dense-base path.
Reflow three wrapped comments in the incremental doc-values update code so phrases no longer break mid-line. Comment-only.
---
.../apache/lucene/index/DocValuesFieldUpdates.java | 4 +-
.../org/apache/lucene/index/ReadersAndUpdates.java | 6 +-
.../lucene/index/SegmentDocValuesProducer.java | 3 +-
.../index/TestIncrementalDocValuesUpdates.java | 91 ++++++++++++++++++++++
4 files changed, 96 insertions(+), 8 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index d4c0662491b..72b119da6a6 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -238,8 +238,8 @@ abstract class DocValuesFieldUpdates implements Accountable {
protected PagedMutable docs;
protected int size;
// true once any doc's value was reset (removed); such a buffer cannot take the sparse incremental
- // path. Protected so
- // subclasses that override reset() (e.g. NumericDocValuesFieldUpdates) can set it too.
+ // path. Protected so subclasses that override reset() (e.g. NumericDocValuesFieldUpdates) can set
+ // it too.
protected boolean anyReset;
protected DocValuesFieldUpdates(int maxDoc, long delGen, String field, DocValuesType type) {
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index adfafd92068..56ec93a3f40 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -439,8 +439,7 @@ final class ReadersAndUpdates {
return DocValuesFieldUpdates.Iterator.asBinaryDocValues(iterator);
}
// sparse fold merges the prior deltas (stays sparse); a dense rewrite (removal or
- // fold-to-dense)
- // merges the base column
+ // fold-to-dense) merges the base column
final BinaryDocValues onDisk =
compact && foldToDense == false
? overlayBinary(deltaProducers, fieldInfo)
@@ -503,8 +502,7 @@ final class ReadersAndUpdates {
return DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator);
}
// sparse fold merges the prior deltas (stays sparse); a dense rewrite (removal or
- // fold-to-dense)
- // merges the base column
+ // fold-to-dense) merges the base column
final NumericDocValues onDisk =
compact && foldToDense == false
? overlayNumeric(deltaProducers, fieldInfo)
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
index a8b37f40baf..b3979856745 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
@@ -200,8 +200,7 @@ class SegmentDocValuesProducer extends DocValuesProducer {
}
// sorted/sorted-set/sorted-numeric and skippers are never overlaid: only numeric/binary values
- // can be updated in
- // place, so these always have a single producer.
+ // can be updated in place, so these always have a single producer.
private DocValuesProducer single(FieldInfo field) {
DocValuesProducer[] producers = dvProducersByField.get(field.number);
assert producers != null && producers.length == 1
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java
index 02e9163eea1..5b2be5e6c29 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java
@@ -514,9 +514,100 @@ public class TestIncrementalDocValuesUpdates extends LuceneTestCase {
assertNumeric(reader, "d" + i, 200L + i);
}
}
+ TestUtil.checkIndex(dir);
}
}
+ /**
+ * Soft deletes are numeric doc-values updates on the soft-deletes field, so they ride the overlay
+ * path too. Marking docs across many commits folds that field's overlay (and folds it to a dense
+ * column once coverage crosses the threshold); liveness and the surviving values must stay
+ * correct throughout.
+ */
+ public void testSoftDeletesOverOverlay() throws Exception {
+ String softField = "__soft_deletes";
+ int numDocs = 20;
+ try (Directory dir = newDirectory();
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setSoftDeletesField(softField)
+ .setMaxDocValuesOverlays(2)
+ .setMergePolicy(NoMergePolicy.INSTANCE))) {
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ w.addDocument(d);
+ }
+ w.commit();
+ // Soft-delete every even doc, one commit each, so the soft-deletes field accrues deltas that
+ // fold and eventually fold to a dense column (coverage reaches half the segment).
+ for (int i = 0; i < numDocs; i += 2) {
+ w.updateDocValues(new Term("id", "d" + i), new NumericDocValuesField(softField, 1L));
+ w.commit();
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ assertEquals(numDocs / 2, reader.numDocs());
+ for (int i = 1; i < numDocs; i += 2) {
+ assertNumeric(reader, "d" + i, i);
+ }
+ }
+ TestUtil.checkIndex(dir);
+ }
+ }
+
+ /**
+ * addIndexes(Directory...) copies segments as-is via copySegmentAsIs, so the copied segment must
+ * keep its doc-values overlay rather than flatten it the way the CodecReader path does.
+ */
+ public void testAddIndexesDirectoryCarriesOverlay() throws Exception {
+ try (Directory src = newDirectory();
+ Directory dst = newDirectory()) {
+ int numDocs = 10;
+ try (IndexWriter w =
+ new IndexWriter(src, incrementalConfig().setMergePolicy(NoMergePolicy.INSTANCE))) {
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ w.addDocument(d);
+ }
+ w.commit();
+ for (int i = 0; i < numDocs; i++) {
+ w.updateNumericDocValue(new Term("id", "d" + i), "val", 100L + i);
+ }
+ w.commit();
+ assertTrue("source segment should carry an overlay", hasOverlay(src));
+ }
+ try (IndexWriter w =
+ new IndexWriter(
+ dst,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMergePolicy(NoMergePolicy.INSTANCE))) {
+ w.addIndexes(src);
+ w.commit();
+ assertTrue("copied segment should still carry the overlay", hasOverlay(dst));
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (int i = 0; i < numDocs; i++) {
+ assertNumeric(reader, "d" + i, 100L + i);
+ }
+ }
+ }
+ TestUtil.checkIndex(dst);
+ }
+ }
+
+ private static boolean hasOverlay(Directory dir) throws IOException {
+ for (SegmentCommitInfo si : SegmentInfos.readLatestCommit(dir)) {
+ if (si.getDocValuesOverlays().isEmpty() == false) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private static void assertOldReaderRejects(Directory dir) throws IOException {
String segmentsFile = SegmentInfos.getLastCommitSegmentsFileName(dir);
try (ChecksumIndexInput in = dir.openChecksumInput(segmentsFile)) {