You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ji...@apache.org on 2026/08/13 07:51:54 UTC
(lucene) branch main updated: Incremental doc-values updates (#16418)
This is an automated email from the ASF dual-hosted git repository.
jimczi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 75eddfaff22 Incremental doc-values updates (#16418)
75eddfaff22 is described below
commit 75eddfaff22eb46e8a68529952fb6787b1258854
Author: Jim Ferenczi <ji...@apache.org>
AuthorDate: Thu Aug 13 09:51:48 2026 +0200
Incremental doc-values updates (#16418)
When you update a doc-values field today, Lucene rewrites the whole column for that field, even if you only touched a few docs. So a tiny change writes a lot, and it gets worse as the segment grows.
The idea is simple: when an update only sets values (no unset), write just the changed docs as a sparse "delta"
generation, and stack the deltas on top of the base column at read time, newest wins. Updating a field becomes
`O(changed docs)` instead of `O(column)`.
Deltas would pile up, so there's a small lifecycle per field:
- every flush writes a delta with only the changed docs
- too many deltas -> fold them into one sparse generation
- deltas end up covering the whole column -> fold back to a single dense column
- a merge flattens everything back to a normal column
Numeric and binary only, set-only (a value unset falls back to the current dense rewrite). No doc-values format
change, the deltas use the codec's existing sparse encoding.
An old Lucene can't read the overlay, so this rolls the segments format version (`VERSION_11_0`), the same way past
segments-file changes did (like the SegmentCommitInfo id in 8.6): every commit written by this version uses it, and an older reader rejects the index up front with `IndexFormatTooNewException` instead of reading a delta as if it was the whole column. Reading a newer index with an older Lucene isn't in the contract anyway (Lucene is forward-only), so nothing is lost. On `main` the feature is on by default; pass `IndexWriterConfig#setMaxDocValuesOverlays(0)` for the classic full-column [...]
that default.
---
lucene/CHANGES.txt | 5 +
.../apache/lucene/index/DocValuesFieldUpdates.java | 10 +
.../lucene/index/DocValuesOverlayMerger.java | 120 ++++
.../java/org/apache/lucene/index/IndexWriter.java | 10 +-
.../org/apache/lucene/index/IndexWriterConfig.java | 30 +
.../apache/lucene/index/LiveIndexWriterConfig.java | 18 +
.../lucene/index/NumericDocValuesFieldUpdates.java | 1 +
.../lucene/index/OverlayBinaryDocValues.java | 63 ++
.../lucene/index/OverlayNumericDocValues.java | 67 ++
.../java/org/apache/lucene/index/ReaderPool.java | 10 +-
.../org/apache/lucene/index/ReadersAndUpdates.java | 199 +++++-
.../org/apache/lucene/index/SegmentCommitInfo.java | 48 ++
.../lucene/index/SegmentDocValuesProducer.java | 167 ++++-
.../java/org/apache/lucene/index/SegmentInfos.java | 42 +-
.../IncrementalDocValuesUpdatesBenchmark.java | 687 +++++++++++++++++++++
.../lucene/index/TestBinaryDocValuesUpdates.java | 6 +-
.../index/TestIncrementalDocValuesUpdates.java | 531 ++++++++++++++++
.../lucene/index/TestNumericDocValuesUpdates.java | 6 +-
.../lucene/index/TestOverlayBinaryDocValues.java | 107 ++++
.../lucene/index/TestOverlayNumericDocValues.java | 165 +++++
.../org/apache/lucene/index/TestReaderPool.java | 32 +-
.../apache/lucene/luke/models/util/IndexUtils.java | 6 +-
.../luke/models/overview/TestOverviewImpl.java | 2 +-
23 files changed, 2269 insertions(+), 63 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 7d648c50b7f..e33ed7cd2b7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -111,6 +111,11 @@ API Changes
New Features
---------------------
+* GITHUB#16418: Incremental doc-values updates. A set-only doc-values update (NUMERIC or BINARY) is written as a sparse
+ "delta" overlay holding just the updated documents and layered over the base column at read time, instead of
+ rewriting the whole column, turning per-update write amplification from O(column) into O(updated docs). Enabled by
+ default; disable with IndexWriterConfig#setMaxDocValuesOverlays(0). (Jim Ferenczi)
+
* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of
modern Hangul conjoining jamo before KoreanTokenizer. (Mingi Jeong)
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index 5fe0d20b25d..d4c0662491b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -237,6 +237,10 @@ abstract class DocValuesFieldUpdates implements Accountable {
protected final int maxDoc;
protected PagedMutable docs;
protected int size;
+ // true once any doc's value was reset (removed); such a buffer cannot take the sparse incremental
+ // path. Protected so
+ // subclasses that override reset() (e.g. NumericDocValuesFieldUpdates) can set it too.
+ protected boolean anyReset;
protected DocValuesFieldUpdates(int maxDoc, long delGen, String field, DocValuesType type) {
this.maxDoc = maxDoc;
@@ -349,9 +353,15 @@ abstract class DocValuesFieldUpdates implements Accountable {
* @param doc the doc to update
*/
synchronized void reset(int doc) {
+ anyReset = true;
addInternal(doc, HAS_NO_VALUE_MASK);
}
+ /** Whether this buffer removed any doc's value (vs. only setting values). */
+ final synchronized boolean anyReset() {
+ return anyReset;
+ }
+
final synchronized int add(int doc) {
return addInternal(doc, HAS_VALUE_MASK);
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesOverlayMerger.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesOverlayMerger.java
new file mode 100644
index 00000000000..4a98517dfb0
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesOverlayMerger.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import org.apache.lucene.search.DocIdSetIterator;
+
+/**
+ * Merges the doc iterators of several doc-values layers (ordered newest first) into the union of
+ * their docs and tracks which layer supplies the current value: the newest layer positioned on the
+ * current doc. Backed by a binary min-heap keyed by (docID, layer index), so advancing only
+ * re-positions the layers sitting on the current doc (the rest keep their place) instead of
+ * scanning every layer per doc. Shared by {@link OverlayNumericDocValues} and {@link
+ * OverlayBinaryDocValues}; layers are advanced with {@code advance} only (never {@code
+ * advanceExact}) so iteration and random access can be interleaved on the same instance.
+ */
+final class DocValuesOverlayMerger {
+
+ private final DocValuesIterator[] layers; // newest first
+ private final int[] heap; // 1-based; holds layer indices, ordered by (docID, index)
+ private final int size;
+ private int docID = -1;
+ private int winner = -1;
+
+ DocValuesOverlayMerger(DocValuesIterator[] layers) {
+ this.layers = layers;
+ this.size = layers.length;
+ this.heap = new int[size + 1];
+ // all layers start at docID -1, so indices in natural order are already a valid min-heap
+ for (int i = 0; i < size; i++) {
+ heap[i + 1] = i;
+ }
+ }
+
+ int docID() {
+ return docID;
+ }
+
+ /** Index of the layer supplying the current value, or -1 if the current doc has no value. */
+ int valueLayer() {
+ return winner;
+ }
+
+ int nextDoc() throws IOException {
+ return advance(docID + 1);
+ }
+
+ int advance(int target) throws IOException {
+ while (layers[heap[1]].docID() < target) {
+ layers[heap[1]].advance(target);
+ siftDown();
+ }
+ docID = layers[heap[1]].docID();
+ winner = docID == DocIdSetIterator.NO_MORE_DOCS ? -1 : heap[1];
+ return docID;
+ }
+
+ boolean advanceExact(int target) throws IOException {
+ while (layers[heap[1]].docID() < target) {
+ layers[heap[1]].advance(target);
+ siftDown();
+ }
+ docID = target;
+ winner = layers[heap[1]].docID() == target ? heap[1] : -1;
+ return winner != -1;
+ }
+
+ long cost() {
+ // Over-estimates the union (a doc in several layers counts once per layer); cost() is only an
+ // upper-bound hint.
+ long cost = 0;
+ for (DocValuesIterator layer : layers) {
+ cost += layer.cost();
+ }
+ return cost;
+ }
+
+ /** Restore the heap after the root layer advanced (its docID only ever increases). */
+ private void siftDown() {
+ int i = 1;
+ int node = heap[1];
+ while (true) {
+ int left = i << 1;
+ if (left > size) {
+ break;
+ }
+ int right = left + 1;
+ int child = right <= size && less(heap[right], heap[left]) ? right : left;
+ if (less(heap[child], node) == false) {
+ break;
+ }
+ heap[i] = heap[child];
+ i = child;
+ }
+ heap[i] = node;
+ }
+
+ private boolean less(int a, int b) {
+ int da = layers[a].docID();
+ int db = layers[b].docID();
+ if (da != db) {
+ return da < db;
+ }
+ return a < b; // same doc: the newer layer (smaller index) wins
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 68ddd1611b9..c4b5dc450af 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1154,7 +1154,8 @@ public class IndexWriter
bufferedUpdatesStream::getCompletedDelGen,
infoStream,
conf.getSoftDeletesField(),
- reader);
+ reader,
+ config);
if (config.getReaderPooling()) {
readerPool.enableReaderPooling();
}
@@ -3624,6 +3625,13 @@ public class IndexWriter
newInfo.setFiles(info.info.files());
newInfoPerCommit.setFieldInfosFiles(info.getFieldInfosFiles());
newInfoPerCommit.setDocValuesUpdatesFiles(info.getDocValuesUpdatesFiles());
+ // Carry over the incremental doc-values overlay generations (the copied update files keep their
+ // gen numbers).
+ for (Map.Entry<Integer, long[]> e : info.getDocValuesOverlays().entrySet()) {
+ final long[] packed = e.getValue();
+ newInfoPerCommit.setDocValuesOverlay(
+ e.getKey(), packed[0], ArrayUtil.copyOfSubArray(packed, 1, packed.length));
+ }
Set<String> copiedFiles = new HashSet<>();
try {
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
index 821a8a1099b..b6b000f64d7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
@@ -107,6 +107,13 @@ public final class IndexWriterConfig extends LiveIndexWriterConfig {
*/
public static final long DEFAULT_MAX_FULL_FLUSH_MERGE_WAIT_MILLIS = 500;
+ /**
+ * Default maximum number of sparse doc-values overlays a field keeps before they are folded into
+ * one: {@code 16}. {@code 0} disables the feature (classic full-column rewrite); higher values
+ * lower write amplification but keep more overlays (hence more files) live to merge at read time.
+ */
+ public static final int DEFAULT_MAX_DOC_VALUES_OVERLAYS = 16;
+
// indicates whether this config instance is already attached to a writer.
// not final so that it can be cloned properly.
private SetOnce<IndexWriter> writer = new SetOnce<>();
@@ -324,6 +331,29 @@ public final class IndexWriterConfig extends LiveIndexWriterConfig {
return readerPooling;
}
+ /**
+ * Sets the maximum number of sparse doc-values overlays a field may accumulate before they are
+ * folded into a single overlay, which also enables or disables the feature. When enabled, a
+ * doc-values update that only sets values (no removals) is written as a sparse "delta" holding
+ * just the updated documents and overlaid on the existing column at read time, rather than
+ * rewriting the whole column. This trades some read cost for much lower write amplification on
+ * frequently updated fields. Pass {@code 0} to disable the feature and keep the classic
+ * full-column rewrite; any value {@code > 0} enables it, keeping up to that many overlays before
+ * a fold. Enabled by default (see {@link #DEFAULT_MAX_DOC_VALUES_OVERLAYS}).
+ *
+ * <p>Only takes effect when IndexWriter is first created.
+ *
+ * @lucene.experimental
+ */
+ public IndexWriterConfig setMaxDocValuesOverlays(int maxDocValuesOverlays) {
+ if (maxDocValuesOverlays < 0) {
+ throw new IllegalArgumentException(
+ "maxDocValuesOverlays must be >= 0; got " + maxDocValuesOverlays);
+ }
+ this.maxDocValuesOverlays = maxDocValuesOverlays;
+ return this;
+ }
+
/**
* Expert: Controls when segments are flushed to disk during indexing. The {@link FlushPolicy}
* initialized during {@link IndexWriter} instantiation and once initialized the given instance is
diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
index f08c86329ea..11a1a5d6edd 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
@@ -90,6 +90,12 @@ public class LiveIndexWriterConfig {
/** True if calls to {@link IndexWriter#close()} should first do a commit. */
protected boolean commitOnClose = IndexWriterConfig.DEFAULT_COMMIT_ON_CLOSE;
+ /**
+ * Maximum number of sparse doc-values overlays a field keeps before they are folded into one;
+ * {@code 0} disables the feature. See {@link IndexWriterConfig#setMaxDocValuesOverlays}.
+ */
+ protected int maxDocValuesOverlays;
+
/** The sort order to use to write merged segments. */
protected Sort indexSort = null;
@@ -137,6 +143,7 @@ public class LiveIndexWriterConfig {
mergePolicy = new TieredMergePolicy();
flushPolicy = new FlushByRamOrCountsPolicy();
readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING;
+ maxDocValuesOverlays = IndexWriterConfig.DEFAULT_MAX_DOC_VALUES_OVERLAYS;
perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB;
maxFullFlushMergeWaitMillis = IndexWriterConfig.DEFAULT_MAX_FULL_FLUSH_MERGE_WAIT_MILLIS;
eventListener = IndexWriterEventListener.NO_OP_LISTENER;
@@ -385,6 +392,16 @@ public class LiveIndexWriterConfig {
return useCompoundFile;
}
+ /**
+ * Returns the maximum number of sparse doc-values overlays a field keeps before they are folded
+ * into one, or {@code 0} if the feature is disabled.
+ *
+ * @lucene.experimental
+ */
+ public int getMaxDocValuesOverlays() {
+ return maxDocValuesOverlays;
+ }
+
/**
* Returns <code>true</code> if {@link IndexWriter#close()} should first commit before closing.
*/
@@ -487,6 +504,7 @@ public class LiveIndexWriterConfig {
sb.append("readerPooling=").append(getReaderPooling()).append("\n");
sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n");
sb.append("useCompoundFile=").append(getUseCompoundFile()).append("\n");
+ sb.append("maxDocValuesOverlays=").append(getMaxDocValuesOverlays()).append("\n");
sb.append("commitOnClose=").append(getCommitOnClose()).append("\n");
sb.append("indexSort=").append(getIndexSort()).append("\n");
sb.append("checkPendingFlushOnUpdate=").append(isCheckPendingFlushOnUpdate()).append("\n");
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
index a3c14486fbd..41b642ddf59 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
@@ -168,6 +168,7 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
@Override
synchronized void reset(int doc) {
+ anyReset = true;
bitSet.set(doc);
this.hasAtLeastOneValue = true;
if (hasNoValue == null) {
diff --git a/lucene/core/src/java/org/apache/lucene/index/OverlayBinaryDocValues.java b/lucene/core/src/java/org/apache/lucene/index/OverlayBinaryDocValues.java
new file mode 100644
index 00000000000..cb6a1f92e0b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/OverlayBinaryDocValues.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import org.apache.lucene.util.BytesRef;
+
+/** Binary counterpart of {@link OverlayNumericDocValues}; see it for the layering and invariant. */
+final class OverlayBinaryDocValues extends BinaryDocValues {
+
+ private final BinaryDocValues[] layers; // newest first, base last
+ private final DocValuesOverlayMerger merger;
+
+ OverlayBinaryDocValues(BinaryDocValues[] layers) {
+ assert layers != null && layers.length > 0;
+ this.layers = layers;
+ this.merger = new DocValuesOverlayMerger(layers);
+ }
+
+ @Override
+ public BytesRef binaryValue() throws IOException {
+ return layers[merger.valueLayer()].binaryValue();
+ }
+
+ @Override
+ public boolean advanceExact(int target) throws IOException {
+ return merger.advanceExact(target);
+ }
+
+ @Override
+ public int docID() {
+ return merger.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return merger.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return merger.advance(target);
+ }
+
+ @Override
+ public long cost() {
+ return merger.cost();
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/OverlayNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/OverlayNumericDocValues.java
new file mode 100644
index 00000000000..af7eccf5583
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/OverlayNumericDocValues.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+
+/**
+ * Merges several {@link NumericDocValues} layers (newest first, base last) into one view: a doc's
+ * value comes from the newest layer that has it, else the base. The delta layers are set-only
+ * (removals go through the dense rewrite), so a doc has a value iff some layer has it and the
+ * newest wins, which makes iteration a plain union-merge.
+ */
+final class OverlayNumericDocValues extends NumericDocValues {
+
+ private final NumericDocValues[] layers; // newest first, base last
+ private final DocValuesOverlayMerger merger;
+
+ OverlayNumericDocValues(NumericDocValues[] layers) {
+ assert layers != null && layers.length > 0;
+ this.layers = layers;
+ this.merger = new DocValuesOverlayMerger(layers);
+ }
+
+ @Override
+ public long longValue() throws IOException {
+ return layers[merger.valueLayer()].longValue();
+ }
+
+ @Override
+ public boolean advanceExact(int target) throws IOException {
+ return merger.advanceExact(target);
+ }
+
+ @Override
+ public int docID() {
+ return merger.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return merger.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return merger.advance(target);
+ }
+
+ @Override
+ public long cost() {
+ return merger.cost();
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReaderPool.java b/lucene/core/src/java/org/apache/lucene/index/ReaderPool.java
index b7ca6634efb..d66b03021c3 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReaderPool.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReaderPool.java
@@ -52,6 +52,7 @@ final class ReaderPool implements Closeable {
private final InfoStream infoStream;
private final SegmentInfos segmentInfos;
private final String softDeletesField;
+ private final LiveIndexWriterConfig config;
// This is a "write once" variable (like the organic dye
// on a DVD-R that may or may not be heated by a laser and
// then cooled to permanently record the event): it's
@@ -75,7 +76,8 @@ final class ReaderPool implements Closeable {
LongSupplier completedDelGenSupplier,
InfoStream infoStream,
String softDeletesField,
- StandardDirectoryReader reader)
+ StandardDirectoryReader reader,
+ LiveIndexWriterConfig config)
throws IOException {
this.directory = directory;
this.originalDirectory = originalDirectory;
@@ -84,6 +86,7 @@ final class ReaderPool implements Closeable {
this.completedDelGenSupplier = completedDelGenSupplier;
this.infoStream = infoStream;
this.softDeletesField = softDeletesField;
+ this.config = config;
if (reader != null) {
// Pre-enroll all segment readers into the reader pool; this is necessary so
// any in-memory NRT live docs are correctly carried over, and so NRT readers
@@ -106,7 +109,8 @@ final class ReaderPool implements Closeable {
new ReadersAndUpdates(
segmentInfos.getIndexCreatedVersionMajor(),
newReader,
- newPendingDeletes(newReader, newReader.getOriginalSegmentInfo())));
+ newPendingDeletes(newReader, newReader.getOriginalSegmentInfo()),
+ config));
}
}
}
@@ -404,7 +408,7 @@ final class ReaderPool implements Closeable {
}
rld =
new ReadersAndUpdates(
- segmentInfos.getIndexCreatedVersionMajor(), info, newPendingDeletes(info));
+ segmentInfos.getIndexCreatedVersionMajor(), info, newPendingDeletes(info), config);
// Steal initial reference:
readerMap.put(info, rld);
} else {
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index 6ba167c6de6..adfafd92068 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
@@ -32,11 +33,13 @@ import java.util.function.Function;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.TrackingDirectoryWrapper;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
@@ -48,6 +51,13 @@ import org.apache.lucene.util.InfoStream;
// searching or merging), plus pending deletes and updates,
// for a given segment
final class ReadersAndUpdates {
+
+ private static final long[] EMPTY_GENS = new long[0];
+
+ // Fold the deltas back into a dense column once they cover at least this fraction of the segment,
+ // capping the read-time overlay depth. TODO: expose as a config option.
+ private static final double FOLD_TO_DENSE_COVERAGE_RATIO = 0.5;
+
// Not final because we replace (clone) when we need to
// change it and it's been shared:
final SegmentCommitInfo info;
@@ -65,6 +75,8 @@ final class ReadersAndUpdates {
// the major version this index was created with
private final int indexCreatedVersionMajor;
+ private final LiveIndexWriterConfig config;
+
// Indicates whether this segment is currently being merged. While a segment
// is merging, all field updates are also registered in the
// mergingDVUpdates map. Also, calls to writeFieldUpdates merge the
@@ -88,10 +100,14 @@ final class ReadersAndUpdates {
final AtomicLong ramBytesUsed = new AtomicLong();
ReadersAndUpdates(
- int indexCreatedVersionMajor, SegmentCommitInfo info, PendingDeletes pendingDeletes) {
+ int indexCreatedVersionMajor,
+ SegmentCommitInfo info,
+ PendingDeletes pendingDeletes,
+ LiveIndexWriterConfig config) {
this.info = info;
this.pendingDeletes = pendingDeletes;
this.indexCreatedVersionMajor = indexCreatedVersionMajor;
+ this.config = config;
}
/**
@@ -100,9 +116,12 @@ final class ReadersAndUpdates {
* <p>NOTE: steals incoming ref from reader.
*/
ReadersAndUpdates(
- int indexCreatedVersionMajor, SegmentReader reader, PendingDeletes pendingDeletes)
+ int indexCreatedVersionMajor,
+ SegmentReader reader,
+ PendingDeletes pendingDeletes,
+ LiveIndexWriterConfig config)
throws IOException {
- this(indexCreatedVersionMajor, reader.getOriginalSegmentInfo(), pendingDeletes);
+ this(indexCreatedVersionMajor, reader.getOriginalSegmentInfo(), pendingDeletes, config);
this.reader = reader;
pendingDeletes.onNewReader(reader, info);
}
@@ -287,6 +306,7 @@ final class ReadersAndUpdates {
DocValuesFormat dvFormat,
final SegmentReader reader,
Map<Integer, Set<String>> fieldFiles,
+ Map<Integer, long[]> newOverlays,
long maxDelGen,
InfoStream infoStream)
throws IOException {
@@ -325,6 +345,66 @@ final class ReadersAndUpdates {
final IOContext updatesContext = IOContext.flush(new FlushInfo(info.info.maxDoc(), bytes));
final FieldInfo fieldInfo = infos.fieldInfo(field);
assert fieldInfo != null;
+ // The field's current overlay {baseGen, deltas...} (or null). baseGen is the recorded base
+ // once the field has deltas, otherwise the field's current column (-1 = core, or an earlier
+ // dense generation).
+ final long[] existingOverlay = info.getDocValuesOverlay(fieldInfo.number);
+ final long baseGen =
+ existingOverlay != null ? existingOverlay[0] : fieldInfo.getDocValuesGen();
+ final long[] priorGens =
+ existingOverlay != null
+ ? ArrayUtil.copyOfSubArray(existingOverlay, 1, existingOverlay.length)
+ : EMPTY_GENS;
+ boolean anyRemoval = false;
+ for (DocValuesFieldUpdates u : updatesToApply) {
+ if (u.anyReset()) {
+ anyRemoval = true;
+ break;
+ }
+ }
+ // Set-only updates become a sparse delta generation overlaid at read time; a removal falls
+ // back to the dense
+ // rewrite. (Skip-indexed fields can't reach here: IndexWriter rejects doc-values updates on
+ // them.)
+ final boolean sparseDelta = config.getMaxDocValuesOverlays() > 0 && anyRemoval == false;
+ // Past maxDocValuesOverlays, fold the prior deltas into this write as one sparse generation
+ // (base untouched) instead of appending another.
+ // TODO: folding all deltas each cycle rewrites the folded generation repeatedly; a
+ // size-tiered policy would help.
+ final boolean compact = sparseDelta && priorGens.length >= config.getMaxDocValuesOverlays();
+ final List<DocValuesProducer> deltaProducers = new ArrayList<>();
+ long deltaCoverage = 0; // sum of the delta generations' docs-with-value counts (>= distinct)
+ // Open the prior deltas to measure their coverage; a sparse fold keeps the base separate
+ // (the core column or an earlier dense generation) and merges only the deltas.
+ if (compact) {
+ try {
+ for (long gen : priorGens) {
+ FieldInfo fiGen = SegmentDocValuesProducer.withGen(fieldInfo, gen);
+ SegmentReadState srs =
+ new SegmentReadState(
+ info.info.dir,
+ info.info,
+ new FieldInfos(new FieldInfo[] {fiGen}),
+ IOContext.DEFAULT,
+ Long.toString(gen, Character.MAX_RADIX));
+ DocValuesProducer p = dvFormat.fieldsProducer(srs);
+ deltaProducers.add(p);
+ deltaCoverage +=
+ type == DocValuesType.BINARY
+ ? p.getBinary(fieldInfo).cost()
+ : p.getNumeric(fieldInfo).cost();
+ }
+ } catch (Throwable t) {
+ // The try-with-resources below owns the normal close; close here if opening a delta fails
+ // first.
+ IOUtils.closeWhileHandlingException(deltaProducers);
+ throw t;
+ }
+ }
+ // Fold into one dense column (reclaiming the base) once the deltas cover at least
+ // FOLD_TO_DENSE_COVERAGE_RATIO of the segment.
+ final boolean foldToDense =
+ compact && deltaCoverage >= info.info.maxDoc() * FOLD_TO_DENSE_COVERAGE_RATIO;
fieldInfo.setDocValuesGen(nextDocValuesGen);
final FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] {fieldInfo});
// separately also track which files were created for this gen
@@ -354,9 +434,20 @@ final class ReadersAndUpdates {
@Override
public BinaryDocValues getBinary(FieldInfo fieldInfoIn) throws IOException {
DocValuesFieldUpdates.Iterator iterator = updateSupplier.apply(fieldInfo);
+ if (sparseDelta && compact == false) {
+ // write only the updated docs; unchanged docs come from the base at read time
+ return DocValuesFieldUpdates.Iterator.asBinaryDocValues(iterator);
+ }
+ // sparse fold merges the prior deltas (stays sparse); a dense rewrite (removal or
+ // fold-to-dense)
+ // merges the base column
+ final BinaryDocValues onDisk =
+ compact && foldToDense == false
+ ? overlayBinary(deltaProducers, fieldInfo)
+ : reader.getBinaryDocValues(field);
final MergedDocValues<BinaryDocValues> mergedDocValues =
new MergedDocValues<>(
- reader.getBinaryDocValues(field),
+ onDisk,
DocValuesFieldUpdates.Iterator.asBinaryDocValues(iterator),
iterator);
// Merge sort of the original doc values with updated doc values:
@@ -407,9 +498,20 @@ final class ReadersAndUpdates {
@Override
public NumericDocValues getNumeric(FieldInfo fieldInfoIn) throws IOException {
DocValuesFieldUpdates.Iterator iterator = updateSupplier.apply(fieldInfo);
+ if (sparseDelta && compact == false) {
+ // write only the updated docs; unchanged docs come from the base at read time
+ return DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator);
+ }
+ // sparse fold merges the prior deltas (stays sparse); a dense rewrite (removal or
+ // fold-to-dense)
+ // merges the base column
+ final NumericDocValues onDisk =
+ compact && foldToDense == false
+ ? overlayNumeric(deltaProducers, fieldInfo)
+ : reader.getNumericDocValues(field);
final MergedDocValues<NumericDocValues> mergedDocValues =
new MergedDocValues<>(
- reader.getNumericDocValues(field),
+ onDisk,
DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator),
iterator);
// Merge sort of the original doc values with updated doc values:
@@ -453,6 +555,26 @@ final class ReadersAndUpdates {
}
});
}
+ } finally {
+ IOUtils.close(deltaProducers);
+ }
+ // Stage the field's overlay generations; writeFieldUpdates commits them to the commit info
+ // only if the whole batch succeeds, so a later field's failure (whose rollback deletes these
+ // gen files) never leaves info referencing them. Packed as {baseGen, deltas...}.
+ if (compact && foldToDense == false) {
+ // the single folded generation replaces the prior deltas (base kept)
+ newOverlays.put(fieldInfo.number, new long[] {baseGen, nextDocValuesGen});
+ } else if (sparseDelta && compact == false) {
+ // prepend this generation to the field's overlay list (base unchanged; -1 = the core
+ // column)
+ final long[] packed = new long[priorGens.length + 2];
+ packed[0] = baseGen;
+ packed[1] = nextDocValuesGen;
+ System.arraycopy(priorGens, 0, packed, 2, priorGens.length);
+ newOverlays.put(fieldInfo.number, packed);
+ } else {
+ // a dense rewrite (removal or fold-to-dense) is a single column again: clear the overlay
+ newOverlays.put(fieldInfo.number, new long[] {-1});
}
info.advanceDocValuesGen();
assert !fieldFiles.containsKey(fieldInfo.number);
@@ -460,6 +582,28 @@ final class ReadersAndUpdates {
}
}
+ /**
+ * Fresh overlay (newest generation first) over the given delta producers, used to fold deltas
+ * during compaction.
+ */
+ private static NumericDocValues overlayNumeric(
+ List<DocValuesProducer> producers, FieldInfo fieldInfo) throws IOException {
+ NumericDocValues[] layers = new NumericDocValues[producers.size()];
+ for (int i = 0; i < layers.length; i++) {
+ layers[i] = producers.get(i).getNumeric(fieldInfo);
+ }
+ return new OverlayNumericDocValues(layers);
+ }
+
+ private static BinaryDocValues overlayBinary(
+ List<DocValuesProducer> producers, FieldInfo fieldInfo) throws IOException {
+ BinaryDocValues[] layers = new BinaryDocValues[producers.size()];
+ for (int i = 0; i < layers.length; i++) {
+ layers[i] = producers.get(i).getBinary(fieldInfo);
+ }
+ return new OverlayBinaryDocValues(layers);
+ }
+
/**
* This class merges the current on-disk DV with an incoming update DV instance and merges the two
* instances giving the incoming update precedence in terms of values, in other words the values
@@ -617,6 +761,9 @@ final class ReadersAndUpdates {
throws IOException {
long startTimeNS = System.nanoTime();
final Map<Integer, Set<String>> newDVFiles = new HashMap<>();
+ // Overlay generations staged per field (committed to info only if the whole write succeeds); it
+ // also drives file retention: a field keeps the files of the generations it still references.
+ final Map<Integer, long[]> newOverlays = new HashMap<>();
Set<String> fieldInfosFiles = null;
FieldInfos fieldInfos = null;
boolean any = false;
@@ -685,7 +832,14 @@ final class ReadersAndUpdates {
final DocValuesFormat docValuesFormat = codec.docValuesFormat();
handleDVUpdates(
- fieldInfos, trackingDir, docValuesFormat, reader, newDVFiles, maxDelGen, infoStream);
+ fieldInfos,
+ trackingDir,
+ docValuesFormat,
+ reader,
+ newDVFiles,
+ newOverlays,
+ maxDelGen,
+ infoStream);
fieldInfosFiles = writeFieldInfosGen(fieldInfos, trackingDir, codec.fieldInfosFormat());
} finally {
@@ -737,18 +891,39 @@ final class ReadersAndUpdates {
assert fieldInfosFiles != null;
info.setFieldInfosFiles(fieldInfosFiles);
- // update the doc-values updates files. the files map each field to its set
- // of files, hence we copy from the existing map all fields w/ updates that
- // were not updated in this session, and add new mappings for fields that
- // were updated now.
+ // A field's live update files are those of the generations its overlay still references (the
+ // core column, baseGen -1, lives in the segment's own files). Carry over untouched fields; for
+ // the rest keep this session's files plus prior files whose generation is still referenced.
assert newDVFiles.isEmpty() == false;
for (Entry<Integer, Set<String>> e : info.getDocValuesUpdatesFiles().entrySet()) {
- if (newDVFiles.containsKey(e.getKey()) == false) {
- newDVFiles.put(e.getKey(), e.getValue());
+ final int field = e.getKey();
+ if (newDVFiles.containsKey(field) == false) {
+ newDVFiles.put(field, e.getValue());
+ continue;
}
+ final long[] overlay = newOverlays.get(field); // {baseGen, deltas...}, or {-1} when cleared
+ final Set<String> live = new HashSet<>(newDVFiles.get(field));
+ for (String f : e.getValue()) {
+ final long gen = IndexFileNames.parseGeneration(f);
+ for (long liveGen : overlay) {
+ if (liveGen != -1 && liveGen == gen) {
+ live.add(f);
+ break;
+ }
+ }
+ }
+ newDVFiles.put(field, live);
}
info.setDocValuesUpdatesFiles(newDVFiles);
+ // Commit the staged overlay generations now that the write and file bookkeeping succeeded, and
+ // before reopening the reader below so it reflects them.
+ for (Entry<Integer, long[]> e : newOverlays.entrySet()) {
+ final long[] packed = e.getValue();
+ info.setDocValuesOverlay(
+ e.getKey(), packed[0], ArrayUtil.copyOfSubArray(packed, 1, packed.length));
+ }
+
// if there is a reader open, reopen it to reflect the updates
if (reader != null) {
swapNewReaderWithLatestLiveDocs();
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
index ed101ecda9e..7a04ed6b99e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
@@ -71,6 +71,11 @@ public class SegmentCommitInfo {
// Track the per-field DocValues update files
private final Map<Integer, Set<String>> dvUpdatesFiles = new HashMap<>();
+ // Per-field sparse doc-values overlay: field number -> {baseGen, deltaGenNewestFirst...}, where a
+ // baseGen of -1 is a sentinel for the core column. Absent for a field using the classic
+ // single-generation column.
+ private final Map<Integer, long[]> dvOverlays = new HashMap<>();
+
// TODO should we add .files() to FieldInfosFormat, like we have on
// LiveDocsFormat?
// track the fieldInfos update files
@@ -134,6 +139,45 @@ public class SegmentCommitInfo {
}
}
+ /**
+ * Per-field incremental doc-values overlay, as {@code field number -> {baseGen,
+ * deltaGenNewestFirst...}}. Empty for segments whose doc-values fields all use the classic
+ * single-generation column.
+ *
+ * @lucene.internal
+ */
+ public Map<Integer, long[]> getDocValuesOverlays() {
+ return Collections.unmodifiableMap(dvOverlays);
+ }
+
+ /**
+ * The {@code {baseGen, deltaGenNewestFirst...}} overlay for a field, or {@code null} if it has
+ * none.
+ */
+ long[] getDocValuesOverlay(int fieldNumber) {
+ return dvOverlays.get(fieldNumber);
+ }
+
+ /**
+ * Records the sparse doc-values overlay generations for a field (newest delta first, over {@code
+ * baseGen}), or clears it when {@code deltaGensNewestFirst} is empty.
+ */
+ void setDocValuesOverlay(int fieldNumber, long baseGen, long[] deltaGensNewestFirst) {
+ if (deltaGensNewestFirst.length == 0) {
+ dvOverlays.remove(fieldNumber);
+ return;
+ }
+ long[] packed = new long[deltaGensNewestFirst.length + 1];
+ packed[0] = baseGen;
+ System.arraycopy(deltaGensNewestFirst, 0, packed, 1, deltaGensNewestFirst.length);
+ dvOverlays.put(fieldNumber, packed);
+ }
+
+ /** True if any field in this commit carries a sparse doc-values overlay. */
+ public boolean hasDocValuesOverlays() {
+ return dvOverlays.isEmpty() == false;
+ }
+
/** Returns the FieldInfos file names. */
public Set<String> getFieldInfosFiles() {
return Collections.unmodifiableSet(fieldInfosFiles);
@@ -397,6 +441,10 @@ public class SegmentCommitInfo {
other.dvUpdatesFiles.put(e.getKey(), new HashSet<>(e.getValue()));
}
+ for (Entry<Integer, long[]> e : dvOverlays.entrySet()) {
+ other.dvOverlays.put(e.getKey(), e.getValue().clone());
+ }
+
other.fieldInfosFiles.addAll(fieldInfosFiles);
return other;
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
index 542325eaefb..a8b37f40baf 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
@@ -18,11 +18,13 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Set;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.internal.hppc.LongArrayList;
+import org.apache.lucene.internal.hppc.LongObjectHashMap;
import org.apache.lucene.store.Directory;
/** Encapsulates multiple producers when there are docvalues updates as one producer */
@@ -31,11 +33,21 @@ import org.apache.lucene.store.Directory;
// producer?
class SegmentDocValuesProducer extends DocValuesProducer {
- final IntObjectHashMap<DocValuesProducer> dvProducersByField = new IntObjectHashMap<>();
+ // one entry per field: the producers for that field, newest generation first, authoritative base
+ // last
+ final IntObjectHashMap<DocValuesProducer[]> dvProducersByField = new IntObjectHashMap<>();
final Set<DocValuesProducer> dvProducers =
Collections.newSetFromMap(new IdentityHashMap<DocValuesProducer, Boolean>());
final LongArrayList dvGens = new LongArrayList();
+ private final SegmentCommitInfo si;
+ private final Directory dir;
+ private final FieldInfos coreInfos;
+ private final SegmentDocValues segDocValues;
+ // producers opened for this reader, keyed by generation, so each generation is opened (and
+ // ref-counted) once
+ private final LongObjectHashMap<DocValuesProducer> openedByGen = new LongObjectHashMap<>();
+
/**
* Creates a new producer that handles updated docvalues fields
*
@@ -52,30 +64,32 @@ class SegmentDocValuesProducer extends DocValuesProducer {
FieldInfos allInfos,
SegmentDocValues segDocValues)
throws IOException {
+ this.si = si;
+ this.dir = dir;
+ this.coreInfos = coreInfos;
+ this.segDocValues = segDocValues;
try {
- DocValuesProducer baseProducer = null;
for (FieldInfo fi : allInfos) {
if (fi.getDocValuesType() == DocValuesType.NONE) {
continue;
}
+ long[] overlay = si.getDocValuesOverlay(fi.number);
+ if (overlay != null) {
+ // incremental update: overlay the sparse delta generations over the base
+ dvProducersByField.put(fi.number, openOverlay(fi, overlay));
+ continue;
+ }
long docValuesGen = fi.getDocValuesGen();
if (docValuesGen == -1) {
- if (baseProducer == null) {
- // the base producer gets the original fieldinfos it wrote
- baseProducer = segDocValues.getDocValuesProducer(docValuesGen, si, dir, coreInfos);
- dvGens.add(docValuesGen);
- dvProducers.add(baseProducer);
- }
- dvProducersByField.put(fi.number, baseProducer);
+ // the base producer gets the original fieldinfos it wrote (shared across all base fields)
+ dvProducersByField.put(fi.number, new DocValuesProducer[] {getProducer(-1, coreInfos)});
} else {
- assert !dvGens.contains(docValuesGen);
// otherwise, producer sees only the one fieldinfo it wrote
- final DocValuesProducer dvp =
- segDocValues.getDocValuesProducer(
- docValuesGen, si, dir, new FieldInfos(new FieldInfo[] {fi}));
- dvGens.add(docValuesGen);
- dvProducers.add(dvp);
- dvProducersByField.put(fi.number, dvp);
+ dvProducersByField.put(
+ fi.number,
+ new DocValuesProducer[] {
+ getProducer(docValuesGen, new FieldInfos(new FieldInfo[] {fi}))
+ });
}
}
} catch (Throwable t) {
@@ -88,46 +102,131 @@ class SegmentDocValuesProducer extends DocValuesProducer {
}
}
+ /**
+ * Opens the producer for one generation, opening (and ref-counting) each generation at most once
+ * per reader.
+ */
+ private DocValuesProducer getProducer(long gen, FieldInfos infos) throws IOException {
+ DocValuesProducer p = openedByGen.get(gen);
+ if (p == null) {
+ p = segDocValues.getDocValuesProducer(gen, si, dir, infos);
+ openedByGen.put(gen, p);
+ dvGens.add(gen);
+ dvProducers.add(p);
+ }
+ return p;
+ }
+
+ /**
+ * Builds the ordered producer stack (newest delta first, base last) for an overlay field from its
+ * {@code {baseGen, deltaGenNewestFirst...}} generations (see {@link
+ * SegmentCommitInfo#getDocValuesOverlay}).
+ */
+ private DocValuesProducer[] openOverlay(FieldInfo fi, long[] overlay) throws IOException {
+ final long baseGen = overlay[0];
+ final int numDeltas = overlay.length - 1;
+ final boolean hasCoreBase = baseGen == -1 && coreInfos.fieldInfo(fi.name) != null;
+ DocValuesProducer[] producers =
+ new DocValuesProducer[numDeltas + ((baseGen != -1 || hasCoreBase) ? 1 : 0)];
+ int i = 0;
+ for (int d = 1; d <= numDeltas; d++) {
+ long gen = overlay[d];
+ producers[i++] = getProducer(gen, new FieldInfos(new FieldInfo[] {withGen(fi, gen)}));
+ }
+ if (baseGen == -1) {
+ if (hasCoreBase) {
+ producers[i] = getProducer(-1, coreInfos);
+ }
+ } else {
+ producers[i] = getProducer(baseGen, new FieldInfos(new FieldInfo[] {withGen(fi, baseGen)}));
+ }
+ return producers;
+ }
+
+ /**
+ * A copy of {@code fi} with its doc-values generation set to {@code gen}, so the codec reads the
+ * right gen files.
+ */
+ static FieldInfo withGen(FieldInfo fi, long gen) {
+ FieldInfo copy =
+ new FieldInfo(
+ fi.name,
+ fi.number,
+ fi.hasTermVectors(),
+ fi.omitsNorms(),
+ fi.hasPayloads(),
+ fi.getIndexOptions(),
+ fi.getDocValuesType(),
+ fi.docValuesSkipIndexType(),
+ gen,
+ new HashMap<>(fi.attributes()),
+ fi.getPointDimensionCount(),
+ fi.getPointIndexDimensionCount(),
+ fi.getPointNumBytes(),
+ fi.getVectorDimension(),
+ fi.getVectorEncoding(),
+ fi.getVectorSimilarityFunction(),
+ fi.isSoftDeletesField(),
+ fi.isParentField());
+ return copy;
+ }
+
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
- DocValuesProducer dvProducer = dvProducersByField.get(field.number);
- assert dvProducer != null;
- return dvProducer.getNumeric(field);
+ DocValuesProducer[] producers = dvProducersByField.get(field.number);
+ assert producers != null;
+ if (producers.length == 1) {
+ return producers[0].getNumeric(field);
+ }
+ NumericDocValues[] layers = new NumericDocValues[producers.length];
+ for (int i = 0; i < producers.length; i++) {
+ layers[i] = producers[i].getNumeric(field);
+ }
+ return new OverlayNumericDocValues(layers);
}
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
- DocValuesProducer dvProducer = dvProducersByField.get(field.number);
- assert dvProducer != null;
- return dvProducer.getBinary(field);
+ DocValuesProducer[] producers = dvProducersByField.get(field.number);
+ assert producers != null;
+ if (producers.length == 1) {
+ return producers[0].getBinary(field);
+ }
+ BinaryDocValues[] layers = new BinaryDocValues[producers.length];
+ for (int i = 0; i < producers.length; i++) {
+ layers[i] = producers[i].getBinary(field);
+ }
+ return new OverlayBinaryDocValues(layers);
+ }
+
+ // sorted/sorted-set/sorted-numeric and skippers are never overlaid: only numeric/binary values
+ // can be updated in
+ // place, so these always have a single producer.
+ private DocValuesProducer single(FieldInfo field) {
+ DocValuesProducer[] producers = dvProducersByField.get(field.number);
+ assert producers != null && producers.length == 1
+ : "field is not a single-generation field: " + field.name;
+ return producers[0];
}
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
- DocValuesProducer dvProducer = dvProducersByField.get(field.number);
- assert dvProducer != null;
- return dvProducer.getSorted(field);
+ return single(field).getSorted(field);
}
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
- DocValuesProducer dvProducer = dvProducersByField.get(field.number);
- assert dvProducer != null;
- return dvProducer.getSortedNumeric(field);
+ return single(field).getSortedNumeric(field);
}
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
- DocValuesProducer dvProducer = dvProducersByField.get(field.number);
- assert dvProducer != null;
- return dvProducer.getSortedSet(field);
+ return single(field).getSortedSet(field);
}
@Override
public DocValuesSkipper getSkipper(FieldInfo field) {
- DocValuesProducer dvProducer = dvProducersByField.get(field.number);
- assert dvProducer != null;
- return dvProducer.getSkipper(field);
+ return single(field).getSkipper(field);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
index 44ef259e12c..a9009952ce5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
@@ -62,7 +62,8 @@ import org.apache.lucene.util.Version;
* <ul>
* <li><code>segments_N</code>: Header, LuceneVersion, Version, NameCounter, SegCount,
* MinSegmentLuceneVersion, <SegName, SegID, SegCodec, DelGen, DeletionCount,
- * FieldInfosGen, DocValuesGen, UpdatesFiles><sup>SegCount</sup>, CommitUserData, Footer
+ * FieldInfosGen, DocValuesGen, UpdatesFiles, DocValuesOverlays><sup>SegCount</sup>,
+ * CommitUserData, Footer
* </ul>
*
* Data types:
@@ -82,6 +83,10 @@ import org.apache.lucene.util.Version;
* <li>CommitUserData --> {@link DataOutput#writeMapOfStrings Map<String,String>}
* <li>UpdatesFiles --> Map<{@link DataOutput#writeInt Int32}, {@link
* DataOutput#writeSetOfStrings(Set) Set<String>}>
+ * <li>DocValuesOverlays --> OverlayCount, <FieldNumber, BaseGen, DeltaCount,
+ * DeltaGen<sup>DeltaCount</sup>><sup>OverlayCount</sup>; OverlayCount, FieldNumber and
+ * DeltaCount are {@link DataOutput#writeInt Int32}, BaseGen and DeltaGen are {@link
+ * DataOutput#writeLong Int64}
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}
* </ul>
*
@@ -106,6 +111,9 @@ import org.apache.lucene.util.Version;
* no updates to DocValues in that segment. Anything above zero means there are updates to
* DocValues stored by {@link DocValuesFormat}.
* <li>UpdatesFiles stores the set of files that were updated in that segment per field.
+ * <li>DocValuesOverlays records, per field that has incremental (sparse) doc-values updates, the
+ * base generation and the delta generations (newest first) overlaid on it at read time; empty
+ * for segments without overlays.
* </ul>
*
* @lucene.experimental
@@ -118,7 +126,10 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
/** The version that recorded SegmentCommitInfo IDs */
public static final int VERSION_86 = 10;
- static final int VERSION_CURRENT = VERSION_86;
+ /** The version that records per-field incremental doc-values overlay generations. */
+ public static final int VERSION_11_0 = 11;
+
+ static final int VERSION_CURRENT = VERSION_11_0;
/** Name of the generation reference file name */
static final String OLD_SEGMENTS_GEN = "segments.gen";
@@ -449,6 +460,19 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
dvUpdateFiles = Collections.unmodifiableMap(map);
}
siPerCommit.setDocValuesUpdatesFiles(dvUpdateFiles);
+ if (format >= VERSION_11_0) {
+ final int numOverlayFields = CodecUtil.readBEInt(input);
+ for (int i = 0; i < numOverlayFields; i++) {
+ final int fieldNumber = CodecUtil.readBEInt(input);
+ final long baseGen = CodecUtil.readBELong(input);
+ final int numDeltas = CodecUtil.readBEInt(input);
+ final long[] deltaGens = new long[numDeltas];
+ for (int g = 0; g < numDeltas; g++) {
+ deltaGens[g] = CodecUtil.readBELong(input);
+ }
+ siPerCommit.setDocValuesOverlay(fieldNumber, baseGen, deltaGens);
+ }
+ }
infos.add(siPerCommit);
Version segmentVersion = info.getVersion();
@@ -694,6 +718,20 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
CodecUtil.writeBEInt(out, e.getKey());
out.writeSetOfStrings(e.getValue());
}
+ // Doc-values overlays, part of the format since VERSION_11_0 (which VERSION_CURRENT always
+ // is).
+ // field -> {baseGen, deltaGenNewestFirst...}; empty for segments without overlays.
+ final Map<Integer, long[]> overlays = siPerCommit.getDocValuesOverlays();
+ CodecUtil.writeBEInt(out, overlays.size());
+ for (Entry<Integer, long[]> e : overlays.entrySet()) {
+ final long[] packed = e.getValue();
+ CodecUtil.writeBEInt(out, e.getKey());
+ CodecUtil.writeBELong(out, packed[0]); // baseGen
+ CodecUtil.writeBEInt(out, packed.length - 1); // number of delta generations
+ for (int g = 1; g < packed.length; g++) {
+ CodecUtil.writeBELong(out, packed[g]);
+ }
+ }
}
out.writeMapOfStrings(userData);
CodecUtil.writeFooter(out);
diff --git a/lucene/core/src/test/org/apache/lucene/index/IncrementalDocValuesUpdatesBenchmark.java b/lucene/core/src/test/org/apache/lucene/index/IncrementalDocValuesUpdatesBenchmark.java
new file mode 100644
index 00000000000..a42b2cb67d0
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/IncrementalDocValuesUpdatesBenchmark.java
@@ -0,0 +1,687 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Locale;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
+import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.ThreadInterruptedException;
+
+/**
+ * Standalone benchmark for incremental (sparse) doc-values updates. This is <b>not</b> a unit test
+ * and does not use the test framework. Run it as a normal Java application (via {@link #main}) so
+ * the JVM is configured like production (real JIT, no forced assertions). It would be removed
+ * before this feature merges; it lives here only so reviewers can reproduce the numbers.
+ *
+ * <p>It compares the three ways to change one field on a document that also carries a vector: a
+ * full reindex of the document ({@code updateDocument}), the classic dense doc-values update
+ * (rewrite the whole column), and the incremental sparse update ({@link
+ * IndexWriterConfig#setMaxDocValuesOverlays}). The index is built and then updated by {@code
+ * dvbench.threads} threads, with a background near-real-time refresh every {@code
+ * dvbench.refreshMs} (like a search application), rather than periodic commits. The same updates
+ * are applied each way and measured identically (update throughput, bytes written per update
+ * against the raw value size, the live file/generation count, and an aggregation scan), and all
+ * three assert the same final aggregate. With {@code dvbench.queryThreads > 0} background threads
+ * run aggregation queries against the refreshing reader; with {@code dvbench.sweep=true} it also
+ * runs the sparse arm across a range of {@code maxDocValuesOverlays}.
+ *
+ * <p>Example (production-like: assertions off, real JIT, mmap directory):
+ *
+ * <pre>
+ * java -da -Xmx6g --enable-native-access=ALL-UNNAMED \
+ * -Ddvbench.docs=5000000 -Ddvbench.dims=512 -Ddvbench.flatVectors=true \
+ * -Ddvbench.updates=1000000 -Ddvbench.threads=4 -Ddvbench.maxGens=16 \
+ * -cp lucene-core.jar:. org.apache.lucene.index.IncrementalDocValuesUpdatesBenchmark
+ * </pre>
+ */
+public class IncrementalDocValuesUpdatesBenchmark {
+
+ private static final int NUM_DOCS = Integer.getInteger("dvbench.docs", 50_000);
+ private static final int DIMS =
+ Integer.getInteger("dvbench.dims", 128); // <= 0 disables the vector
+ private static final int NUM_UPDATES = Integer.getInteger("dvbench.updates", 100_000);
+ private static final int THREADS = Integer.getInteger("dvbench.threads", 1);
+ private static final int REFRESH_MS = Integer.getInteger("dvbench.refreshMs", 1_000);
+ private static final int MAX_GENS = Integer.getInteger("dvbench.maxGens", 16);
+ private static final String TYPE =
+ System.getProperty("dvbench.type", "numeric"); // numeric, binary, softdelete
+ private static final boolean BINARY = "binary".equals(TYPE);
+ private static final boolean SOFT_DELETES = "softdelete".equals(TYPE);
+ private static final int BINLEN =
+ Integer.getInteger("dvbench.binlen", 32); // binary value length, >= 8
+ private static final boolean FLAT_VECTORS = Boolean.getBoolean("dvbench.flatVectors");
+ private static final int QUERY_THREADS = Integer.getInteger("dvbench.queryThreads", 0);
+ // Aggregate updates/sec across all threads; 0 = as fast as possible (batch). A fixed rate models
+ // a steady stream,
+ // where dense rewrites the whole column on every refresh regardless of how few docs actually
+ // changed.
+ private static final double RATE = Double.parseDouble(System.getProperty("dvbench.rate", "0"));
+ // Random-access lookup samples every Nth doc (in ascending docid order, the sort/facet read
+ // pattern); default 10 = ~10% of docs.
+ private static final int LOOKUP_STRIDE = Integer.getInteger("dvbench.lookupStride", 10);
+ private static final String SOFT_FIELD = "__soft_delete";
+
+ /** Sink so the JIT cannot elide the random-access lookup whose result is otherwise unused. */
+ @SuppressWarnings("unused")
+ private static volatile long blackhole;
+
+ /**
+ * Raw size of one logical value: what a single update actually changes, the denominator for write
+ * amplification.
+ */
+ private static final int RAW_VALUE_BYTES = BINARY ? BINLEN : Long.BYTES;
+
+ private static final int VECTOR_BYTES = DIMS > 0 ? DIMS * Float.BYTES : 0;
+
+ /**
+ * No field is analyzed (the id is a StringField), so a no-op analyzer avoids pulling in an
+ * analysis module.
+ */
+ private static Analyzer noopAnalyzer() {
+ return new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ throw new UnsupportedOperationException("no analyzed fields in this benchmark");
+ }
+ };
+ }
+
+ /** Flat (graph-less) vector storage so million-doc runs at high dimensions stay tractable. */
+ private static Codec flatVectorCodec() {
+ return new FilterCodec("Lucene104", Codec.forName("Lucene104")) {
+ @Override
+ public KnnVectorsFormat knnVectorsFormat() {
+ return new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE);
+ }
+ };
+ }
+
+ private enum Mode {
+ FULL_REINDEX, // updateDocument: rewrite the whole document
+ DENSE_DV, // update, feature off: rewrite the whole column
+ SPARSE_DV // update, feature on: sparse delta generation
+ }
+
+ private static final class CountingDirectory extends FilterDirectory {
+ final AtomicLong bytesWritten = new AtomicLong();
+
+ CountingDirectory(Directory in) {
+ super(in);
+ }
+
+ @Override
+ public IndexOutput createOutput(String name, IOContext context) throws IOException {
+ return new CountingOutput(super.createOutput(name, context), bytesWritten);
+ }
+
+ @Override
+ public IndexOutput createTempOutput(String prefix, String suffix, IOContext context)
+ throws IOException {
+ return new CountingOutput(super.createTempOutput(prefix, suffix, context), bytesWritten);
+ }
+ }
+
+ private static final class CountingOutput extends IndexOutput {
+ private final IndexOutput in;
+ private final AtomicLong counter;
+
+ CountingOutput(IndexOutput in, AtomicLong counter) {
+ super("Counting(" + in + ")", in.getName());
+ this.in = in;
+ this.counter = counter;
+ }
+
+ @Override
+ public void writeByte(byte b) throws IOException {
+ counter.incrementAndGet();
+ in.writeByte(b);
+ }
+
+ @Override
+ public void writeBytes(byte[] b, int offset, int length) throws IOException {
+ counter.addAndGet(length);
+ in.writeBytes(b, offset, length);
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public long getFilePointer() {
+ return in.getFilePointer();
+ }
+
+ @Override
+ public long getChecksum() throws IOException {
+ return in.getChecksum();
+ }
+ }
+
+ private static float[] vectorForId(int id) {
+ if (DIMS <= 0) {
+ return null;
+ }
+ Random rnd = new Random(id);
+ float[] v = new float[DIMS];
+ double norm = 0;
+ for (int i = 0; i < DIMS; i++) {
+ v[i] = rnd.nextFloat();
+ norm += v[i] * v[i];
+ }
+ norm = Math.sqrt(norm);
+ for (int i = 0; i < DIMS; i++) {
+ v[i] /= (float) norm;
+ }
+ return v;
+ }
+
+ /**
+ * Encodes the logical long value into a {@code BINLEN}-byte payload so the binary arm aggregates
+ * the same sum.
+ */
+ private static BytesRef toBytes(long v) {
+ byte[] b = new byte[BINLEN];
+ for (int i = 0; i < BINLEN; i++) {
+ b[i] = (byte) (v >>> (8 * (i % 8)));
+ }
+ return new BytesRef(b);
+ }
+
+ private static long fromBytes(BytesRef ref) {
+ long v = 0;
+ for (int i = 7; i >= 0; i--) {
+ v = (v << 8) | (ref.bytes[ref.offset + i] & 0xFFL);
+ }
+ return v;
+ }
+
+ /**
+ * Rebuilds the full document for {@code id} with the given value (used by the full-reindex arm).
+ */
+ private static Document docFor(int id, long val) {
+ Document d = new Document();
+ d.add(new StringField("id", Integer.toString(id), StringField.Store.NO));
+ if (BINARY) {
+ d.add(new BinaryDocValuesField("val", toBytes(val)));
+ } else {
+ d.add(new NumericDocValuesField("val", val));
+ }
+ // Soft-delete docs start live with no soft-delete field: a doc is soft-deleted once the field
+ // is
+ // present on it, so the mark is added later as a doc-values update, never at build time.
+ if (DIMS > 0) {
+ d.add(new KnnFloatVectorField("vec", vectorForId(id), VectorSimilarityFunction.DOT_PRODUCT));
+ }
+ return d;
+ }
+
+ private static void applyUpdate(IndexWriter w, Mode mode, int id, long val) throws IOException {
+ Term term = new Term("id", Integer.toString(id));
+ if (SOFT_DELETES) {
+ // The soft-delete mark is itself a numeric doc-values update, so the dense/sparse arms
+ // measure exactly what this
+ // feature changes about soft deletes; the full-reindex slot is a hard delete (liveDocs, no
+ // doc-values write).
+ if (mode == Mode.FULL_REINDEX) {
+ w.deleteDocuments(term);
+ } else {
+ // updateDocValues creates the soft-delete column on first use; marking a doc sets it to 1.
+ w.updateDocValues(term, new NumericDocValuesField(SOFT_FIELD, 1L));
+ }
+ } else if (mode == Mode.FULL_REINDEX) {
+ w.updateDocument(term, docFor(id, val));
+ } else if (BINARY) {
+ w.updateBinaryDocValue(term, "val", toBytes(val));
+ } else {
+ w.updateNumericDocValue(term, "val", val);
+ }
+ }
+
+ private record ArmResult(
+ long updateBytes,
+ long updateMs,
+ long scanMs,
+ long lookupMs,
+ long sum,
+ int peakDvd,
+ int endFiles,
+ long queries,
+ double avgQueryMs) {}
+
+ private interface ShardTask {
+ void run(int shard) throws Exception;
+ }
+
+ /**
+ * Runs {@code task} on {@code threads} threads (shards 0..threads-1) and waits for all of them.
+ */
+ private static void parallel(int threads, AtomicReference<Throwable> error, ShardTask task) {
+ Thread[] workers = new Thread[threads];
+ for (int i = 0; i < threads; i++) {
+ final int shard = i;
+ workers[i] =
+ new Thread(
+ () -> {
+ try {
+ task.run(shard);
+ } catch (Throwable t) {
+ error.compareAndSet(null, t);
+ }
+ });
+ workers[i].start();
+ }
+ for (Thread t : workers) {
+ try {
+ t.join();
+ } catch (InterruptedException ie) {
+ throw new ThreadInterruptedException(ie);
+ }
+ }
+ }
+
+ /**
+ * Counts {@code .dvd} data files in the directory, one per live doc-values generation (base +
+ * deltas).
+ */
+ private static int countDvd(Directory dir) throws IOException {
+ int n = 0;
+ for (String f : dir.listAll()) {
+ if (f.endsWith(".dvd")) {
+ n++;
+ }
+ }
+ return n;
+ }
+
+ /** Live-doc aggregation over the updated field, the read path the overlay affects. */
+ private static long aggScan(DirectoryReader reader) throws IOException {
+ long sum = 0;
+ for (LeafReaderContext ctx : reader.leaves()) {
+ Bits live = ctx.reader().getLiveDocs();
+ if (BINARY) {
+ BinaryDocValues dv = ctx.reader().getBinaryDocValues("val");
+ for (int doc = dv.nextDoc(); doc != BinaryDocValues.NO_MORE_DOCS; doc = dv.nextDoc()) {
+ if (live == null || live.get(doc)) {
+ sum += fromBytes(dv.binaryValue());
+ }
+ }
+ } else {
+ NumericDocValues dv = ctx.reader().getNumericDocValues("val");
+ // In soft-delete mode a doc is excluded when its soft-delete mark is set (reading that mark
+ // exercises the
+ // overlay, which is the point). The mark field is absent on the hard-delete arm, so soft
+ // stays null there.
+ NumericDocValues soft = SOFT_DELETES ? ctx.reader().getNumericDocValues(SOFT_FIELD) : null;
+ for (int doc = dv.nextDoc(); doc != NumericDocValues.NO_MORE_DOCS; doc = dv.nextDoc()) {
+ if (live != null && live.get(doc) == false) {
+ continue;
+ }
+ if (soft != null && soft.advanceExact(doc) && soft.longValue() != 0) {
+ continue;
+ }
+ sum += dv.longValue();
+ }
+ }
+ }
+ return sum;
+ }
+
+ /**
+ * Random-access doc-values lookup over an ascending sample of docs, the sort/facet read pattern
+ * (forward-only {@code advanceExact}, as the doc-values contract requires). This is where the
+ * overlay merge is felt most, since each lookup resolves the doc across all layers rather than
+ * streaming.
+ */
+ private static long randomLookup(DirectoryReader reader) throws IOException {
+ long checksum = 0;
+ for (LeafReaderContext ctx : reader.leaves()) {
+ int maxDoc = ctx.reader().maxDoc();
+ if (BINARY) {
+ BinaryDocValues dv = ctx.reader().getBinaryDocValues("val");
+ for (int doc = 0; doc < maxDoc; doc += LOOKUP_STRIDE) {
+ if (dv.advanceExact(doc)) {
+ checksum += fromBytes(dv.binaryValue());
+ }
+ }
+ } else {
+ NumericDocValues dv = ctx.reader().getNumericDocValues("val");
+ for (int doc = 0; doc < maxDoc; doc += LOOKUP_STRIDE) {
+ if (dv.advanceExact(doc)) {
+ checksum += dv.longValue();
+ }
+ }
+ }
+ }
+ return checksum;
+ }
+
+ /** Runs aggregation queries against the refreshing reader until stopped. */
+ private static final class Querier extends Thread {
+ private final ReaderManager readers;
+ private final AtomicBoolean stop;
+ private final AtomicReference<Throwable> error;
+ long queries;
+ long totalNanos;
+
+ Querier(ReaderManager readers, AtomicBoolean stop, AtomicReference<Throwable> error) {
+ this.readers = readers;
+ this.stop = stop;
+ this.error = error;
+ }
+
+ @Override
+ public void run() {
+ try {
+ while (stop.get() == false) {
+ DirectoryReader reader = readers.acquire();
+ try {
+ long t0 = System.nanoTime();
+ aggScan(reader);
+ totalNanos += System.nanoTime() - t0;
+ queries++;
+ } finally {
+ readers.release(reader);
+ }
+ }
+ } catch (Throwable t) {
+ error.compareAndSet(null, t);
+ }
+ }
+ }
+
+ private static ArmResult runArm(Mode mode, int maxGens) throws IOException {
+ Path tmp = Files.createTempDirectory("dvbench");
+ CountingDirectory dir = new CountingDirectory(FSDirectory.open(tmp));
+ IndexWriterConfig conf =
+ new IndexWriterConfig(noopAnalyzer())
+ .setMaxDocValuesOverlays(mode == Mode.SPARSE_DV ? maxGens : 0);
+ if (FLAT_VECTORS && DIMS > 0) {
+ conf.setCodec(flatVectorCodec());
+ }
+ if (SOFT_DELETES) {
+ conf.setSoftDeletesField(SOFT_FIELD);
+ }
+ IndexWriter w = new IndexWriter(dir, conf);
+ AtomicReference<Throwable> error = new AtomicReference<>();
+
+ parallel(
+ THREADS,
+ error,
+ shard -> {
+ for (int i = shard; i < NUM_DOCS; i += THREADS) {
+ w.addDocument(docFor(i, i));
+ }
+ });
+ rethrow(error);
+ w.commit();
+ // Let the natural background merges finish; updates run against the many-segment index they
+ // leave.
+ w.waitForMerges();
+
+ // Pre-generate the updates. The ids are uniformly random, so updates hit scattered docs in
+ // random order (not the
+ // ingestion order); each delta generation then holds non-contiguous docids, which stresses the
+ // sparse encoding
+ // and the overlay. Pre-generating also lets every arm and thread count reach the same final
+ // values: each doc's
+ // updates all run on the same thread (partitioned by id), so the last-writer-wins result is
+ // deterministic.
+ int[] ids = new int[NUM_UPDATES];
+ long[] vals = new long[NUM_UPDATES];
+ Random up = new Random(7);
+ for (int i = 0; i < NUM_UPDATES; i++) {
+ ids[i] = up.nextInt(NUM_DOCS);
+ vals[i] = up.nextLong();
+ }
+
+ ReaderManager readers = new ReaderManager(w);
+ AtomicBoolean stop = new AtomicBoolean();
+ AtomicInteger peakDvd = new AtomicInteger();
+
+ // Near-real-time refresh: reopen every REFRESH_MS, which applies the buffered doc-values
+ // updates (writing their
+ // generations), the way a search application refreshes rather than committing on a fixed update
+ // count.
+ Thread refresher =
+ new Thread(
+ () -> {
+ try {
+ while (stop.get() == false) {
+ LockSupport.parkNanos(REFRESH_MS * 1_000_000L);
+ readers.maybeRefresh();
+ peakDvd.accumulateAndGet(countDvd(dir), Math::max);
+ }
+ } catch (Throwable t) {
+ error.compareAndSet(null, t);
+ }
+ });
+ refresher.start();
+
+ Querier[] queriers = new Querier[QUERY_THREADS];
+ for (int i = 0; i < QUERY_THREADS; i++) {
+ queriers[i] = new Querier(readers, stop, error);
+ queriers[i].start();
+ }
+
+ long startWrite = dir.bytesWritten.get();
+ final long updateStart = System.nanoTime();
+ // With a target rate, each thread paces itself to RATE/THREADS updates/sec so the aggregate
+ // matches RATE.
+ final long perThreadIntervalNanos = RATE > 0 ? (long) (1e9 * THREADS / RATE) : 0;
+ parallel(
+ THREADS,
+ error,
+ shard -> {
+ int applied = 0;
+ for (int i = 0; i < NUM_UPDATES; i++) {
+ if (ids[i] % THREADS != shard) {
+ continue;
+ }
+ if (perThreadIntervalNanos > 0) {
+ long wait = (updateStart + applied * perThreadIntervalNanos) - System.nanoTime();
+ if (wait > 0) {
+ LockSupport.parkNanos(wait);
+ }
+ }
+ applyUpdate(w, mode, ids[i], vals[i]);
+ applied++;
+ }
+ });
+ long updateMs = (System.nanoTime() - updateStart) / 1_000_000;
+
+ stop.set(
+ true); // the refresher finishes its current sleep + refresh, then exits (no interrupt: it
+ // may be mid-IO)
+ join(refresher);
+ long queries = 0;
+ long queryNanos = 0;
+ for (Querier q : queriers) {
+ join(q);
+ queries += q.queries;
+ queryNanos += q.totalNanos;
+ }
+ rethrow(error);
+
+ readers.maybeRefresh(); // apply any updates buffered since the last refresh
+ peakDvd.accumulateAndGet(countDvd(dir), Math::max);
+ long updateBytes = dir.bytesWritten.get() - startWrite;
+ int endFiles = dir.listAll().length;
+
+ DirectoryReader reader = readers.acquire();
+ long sum;
+ long scanMs;
+ long lookupMs;
+ try {
+ long scanStart = System.nanoTime();
+ sum = aggScan(reader);
+ scanMs = (System.nanoTime() - scanStart) / 1_000_000;
+ long lookupStart = System.nanoTime();
+ blackhole += randomLookup(reader);
+ lookupMs = (System.nanoTime() - lookupStart) / 1_000_000;
+ } finally {
+ readers.release(reader);
+ }
+
+ readers.close();
+ w.close();
+ dir.close();
+ IOUtils.rm(tmp);
+ double avgQueryMs = queries == 0 ? 0 : queryNanos / (double) queries / 1e6;
+ return new ArmResult(
+ updateBytes, updateMs, scanMs, lookupMs, sum, peakDvd.get(), endFiles, queries, avgQueryMs);
+ }
+
+ private static void join(Thread t) {
+ try {
+ t.join();
+ } catch (InterruptedException ie) {
+ throw new ThreadInterruptedException(ie);
+ }
+ }
+
+ private static void rethrow(AtomicReference<Throwable> error) throws IOException {
+ Throwable t = error.get();
+ if (t != null) {
+ throw new RuntimeException("benchmark worker failed", t);
+ }
+ }
+
+ private static void report(String label, ArmResult r) {
+ double perUpdate = (double) r.updateBytes / NUM_UPDATES;
+ double updatesPerSec = r.updateMs == 0 ? 0 : NUM_UPDATES / (r.updateMs / 1000.0);
+ String concurrent =
+ r.queries == 0
+ ? ""
+ : String.format(Locale.ROOT, " concurrent_q=%d@%,.1fms", r.queries, r.avgQueryMs);
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "[%-12s] %,.0f upd/s per_update=%,.0fB amp=%,.1fx(raw %dB) total=%,.1fMB peak_dv_gens=%d files=%d scan=%dms lookup=%dms%s",
+ label,
+ updatesPerSec,
+ perUpdate,
+ perUpdate / RAW_VALUE_BYTES,
+ RAW_VALUE_BYTES,
+ r.updateBytes / 1e6,
+ r.peakDvd,
+ r.endFiles,
+ r.scanMs,
+ r.lookupMs,
+ concurrent));
+ }
+
+ private static void checkSame(long a, long b) {
+ if (a != b) {
+ throw new AssertionError("arms disagree on the final aggregate: " + a + " != " + b);
+ }
+ }
+
+ private static String rateStr() {
+ return RATE > 0 ? String.format(Locale.ROOT, "%,.0f/s", RATE) : "unlimited";
+ }
+
+ private static void runArms() throws IOException {
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "=== %s val=%dB, %d docs, vector=%dd/%dB (%s), doc~%dB, %d updates, %d threads, rate=%s, refresh/%dms, maxGens=%d, queryThreads=%d ===",
+ TYPE.toUpperCase(Locale.ROOT),
+ RAW_VALUE_BYTES,
+ NUM_DOCS,
+ DIMS,
+ VECTOR_BYTES,
+ DIMS <= 0 ? "none" : (FLAT_VECTORS ? "flat" : "hnsw"),
+ RAW_VALUE_BYTES + VECTOR_BYTES,
+ NUM_UPDATES,
+ THREADS,
+ rateStr(),
+ REFRESH_MS,
+ MAX_GENS,
+ QUERY_THREADS));
+ ArmResult a = runArm(Mode.FULL_REINDEX, MAX_GENS);
+ report(SOFT_DELETES ? "hard_delete" : "full_reindex", a);
+ ArmResult b = runArm(Mode.DENSE_DV, MAX_GENS);
+ report(SOFT_DELETES ? "dense_soft" : "dense_dv", b);
+ ArmResult c = runArm(Mode.SPARSE_DV, MAX_GENS);
+ report(SOFT_DELETES ? "sparse_soft" : "sparse_dv", c);
+ checkSame(a.sum, b.sum);
+ checkSame(b.sum, c.sum);
+ }
+
+ private static void runSweep() throws IOException {
+ System.out.println(
+ String.format(
+ Locale.ROOT,
+ "=== SWEEP %s: %d docs, %d updates, %d threads, refresh/%dms ===",
+ BINARY ? "BINARY" : "NUMERIC",
+ NUM_DOCS,
+ NUM_UPDATES,
+ THREADS,
+ REFRESH_MS));
+ long expect = Long.MIN_VALUE;
+ for (int k : new int[] {2, 4, 8, 16, 32, 64, 128}) {
+ ArmResult r = runArm(Mode.SPARSE_DV, k);
+ report("sparse maxGens=" + k, r);
+ if (expect == Long.MIN_VALUE) {
+ expect = r.sum;
+ } else {
+ checkSame(expect, r.sum);
+ }
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ boolean assertionsOn = false;
+ assert (assertionsOn = true) == true; // Intentional side-effect
+ if (assertionsOn) {
+ System.out.println(
+ "WARNING: assertions are enabled (-ea); re-run with -da for production-like timing.");
+ }
+ runArms();
+ if (Boolean.getBoolean("dvbench.sweep")) {
+ runSweep();
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
index e014f69275a..d28dfe1f702 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
@@ -1326,7 +1326,11 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
public void testDeleteUnusedUpdatesFiles() throws Exception {
Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ // Asserts the dense-rewrite behavior of superseding a field's prior generation, so it disables
+ // the sparse incremental path, which intentionally retains prior overlays to layer them at read
+ // time.
+ IndexWriterConfig conf =
+ newIndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(0);
IndexWriter writer = new IndexWriter(dir, conf);
Document doc = new Document();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java
new file mode 100644
index 00000000000..02e9163eea1
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java
@@ -0,0 +1,531 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Tests the incremental doc-values update path ({@link IndexWriterConfig#setMaxDocValuesOverlays}),
+ * where a set-only update is stored as a sparse delta overlaid on the base column at read time and
+ * the overlays are folded once they exceed the configured maximum.
+ */
+public class TestIncrementalDocValuesUpdates extends LuceneTestCase {
+
+ private IndexWriterConfig incrementalConfig() {
+ return new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxDocValuesOverlays(TestUtil.nextInt(random(), 1, 6));
+ }
+
+ private static void assertNumeric(IndexReader reader, String id, long expected)
+ throws IOException {
+ assertNumericField(reader, id, "val", expected);
+ }
+
+ private static void assertNumericField(IndexReader reader, String id, String field, long expected)
+ throws IOException {
+ for (LeafReaderContext ctx : reader.leaves()) {
+ TermsEnum te = ctx.reader().terms("id").iterator();
+ if (te.seekExact(new BytesRef(id))) {
+ PostingsEnum pe = te.postings(null);
+ int doc = pe.nextDoc();
+ NumericDocValues dv = ctx.reader().getNumericDocValues(field);
+ assertTrue(id + "." + field, dv.advanceExact(doc));
+ assertEquals(id + "." + field, expected, dv.longValue());
+ return;
+ }
+ }
+ fail("id not found: " + id);
+ }
+
+ /** Random set-only updates across many docs and fields, interleaved with reopens and merges. */
+ public void testRandomizedSetOnlyUpdates() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ int numDocs = atLeast(50);
+ Map<String, Long> expected = new HashMap<>();
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ expected.put("d" + i, (long) i);
+ w.addDocument(d);
+ }
+ int updates = atLeast(200);
+ for (int i = 0; i < updates; i++) {
+ String id = "d" + random().nextInt(numDocs);
+ long v = random().nextLong();
+ w.updateNumericDocValue(new Term("id", id), "val", v);
+ expected.put(id, v);
+ if (rarely()) {
+ w.commit();
+ }
+ if (rarely()) {
+ w.forceMerge(1 + random().nextInt(3));
+ }
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (Map.Entry<String, Long> e : expected.entrySet()) {
+ assertNumeric(reader, e.getKey(), e.getValue());
+ }
+ }
+ }
+ }
+
+ /**
+ * A merge flattens the overlay back into a single dense column that still reads the latest
+ * values.
+ */
+ public void testMergeFlattensOverlay() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ for (int i = 0; i < 20; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ w.addDocument(d);
+ }
+ for (int round = 0; round < 10; round++) {
+ for (int i = 0; i < 20; i++) {
+ w.updateNumericDocValue(new Term("id", "d" + i), "val", 1000L + round * 100 + i);
+ }
+ }
+ w.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ assertEquals(1, reader.leaves().size());
+ for (int i = 0; i < 20; i++) {
+ assertNumeric(reader, "d" + i, 1000L + 9 * 100 + i);
+ }
+ }
+ }
+ }
+
+ /**
+ * Removing a value (null update) falls back to the dense rewrite and is observed as "no value".
+ */
+ public void testResetRemovesValue() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ Document d = new Document();
+ d.add(new StringField("id", "d0", StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", 5));
+ w.addDocument(d);
+ w.updateNumericDocValue(new Term("id", "d0"), "val", 7);
+ w.updateDocValues(new Term("id", "d0"), new NumericDocValuesField("val", null));
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ LeafReaderContext ctx = reader.leaves().get(0);
+ NumericDocValues dv = ctx.reader().getNumericDocValues("val");
+ assertFalse("value should have been removed", dv.advanceExact(0));
+ }
+ }
+ }
+
+ /**
+ * An index whose updates were written with the feature disabled keeps working when it is enabled.
+ */
+ public void testContinuesIndexWrittenWithFeatureDisabled() throws Exception {
+ try (Directory dir = newDirectory()) {
+ try (IndexWriter w =
+ new IndexWriter(
+ dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(0))) {
+ for (int i = 0; i < 10; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ w.addDocument(d);
+ }
+ // dense update generations, written by the classic path
+ for (int i = 0; i < 10; i++) {
+ w.updateNumericDocValue(new Term("id", "d" + i), "val", 100L + i);
+ }
+ w.commit();
+ }
+ // reopen with the feature enabled and stack sparse deltas on top of the dense generations
+ try (IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ for (int i = 0; i < 10; i++) {
+ w.updateNumericDocValue(new Term("id", "d" + i), "val", 200L + i);
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (int i = 0; i < 10; i++) {
+ assertNumeric(reader, "d" + i, 200L + i);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Repeatedly updating the whole corpus makes the delta generations cover the entire column; the
+ * writer then folds back to a single dense column (reclaiming the base) instead of overlaying an
+ * ever-denser delta. Checked here only for value correctness across the transition.
+ */
+ public void testFoldsToDenseOnFullCorpusUpdates() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(2))) {
+ int numDocs = 40;
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ w.addDocument(d);
+ }
+ long expected = 0;
+ for (int round = 0; round < 12; round++) {
+ expected = 1000L + round;
+ for (int i = 0; i < numDocs; i++) {
+ w.updateNumericDocValue(new Term("id", "d" + i), "val", expected + i);
+ }
+ w.commit();
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (int i = 0; i < numDocs; i++) {
+ assertNumeric(reader, "d" + i, expected + i);
+ }
+ }
+ }
+ }
+
+ /**
+ * Several updatable fields on the same docs, updated independently. Each field's overlay is
+ * tracked separately.
+ */
+ public void testMultipleUpdatableFields() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ int numDocs = atLeast(40);
+ int numFields = 3;
+ long[][] expected = new long[numDocs][numFields];
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ for (int f = 0; f < numFields; f++) {
+ long v = i * 10L + f;
+ d.add(new NumericDocValuesField("f" + f, v));
+ expected[i][f] = v;
+ }
+ w.addDocument(d);
+ }
+ int updates = atLeast(300);
+ for (int u = 0; u < updates; u++) {
+ int i = random().nextInt(numDocs);
+ int f = random().nextInt(numFields);
+ long v = random().nextLong();
+ w.updateNumericDocValue(new Term("id", "d" + i), "f" + f, v);
+ expected[i][f] = v;
+ if (rarely()) {
+ w.commit();
+ }
+ if (rarely()) {
+ w.forceMerge(1);
+ }
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (int i = 0; i < numDocs; i++) {
+ for (int f = 0; f < numFields; f++) {
+ assertNumericField(reader, "d" + i, "f" + f, expected[i][f]);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Updates interleaved with deletes: deleted docs drop out, surviving docs keep their latest
+ * value.
+ */
+ public void testUpdatesInterleavedWithDeletes() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ int numDocs = atLeast(60);
+ Map<String, Long> expected = new HashMap<>();
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ expected.put("d" + i, (long) i);
+ w.addDocument(d);
+ }
+ int ops = atLeast(300);
+ for (int o = 0; o < ops; o++) {
+ String id = "d" + random().nextInt(numDocs);
+ if (expected.containsKey(id) && random().nextInt(6) == 0) {
+ w.deleteDocuments(new Term("id", id));
+ expected.remove(id);
+ } else if (expected.containsKey(id)) {
+ long v = random().nextLong();
+ w.updateNumericDocValue(new Term("id", id), "val", v);
+ expected.put(id, v);
+ }
+ if (rarely()) {
+ w.commit();
+ }
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (Map.Entry<String, Long> e : expected.entrySet()) {
+ assertNumeric(reader, e.getKey(), e.getValue());
+ }
+ }
+ }
+ }
+
+ /**
+ * Doc-values updates on an index that is sorted on a different field: overlay docs are in the
+ * sorted order.
+ */
+ public void testSortedIndexWithUpdates() throws Exception {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig conf =
+ incrementalConfig().setIndexSort(new Sort(new SortField("sortkey", SortField.Type.LONG)));
+ try (IndexWriter w = new IndexWriter(dir, conf)) {
+ int numDocs = atLeast(40);
+ Map<String, Long> expected = new HashMap<>();
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("sortkey", random().nextInt(1000)));
+ d.add(new NumericDocValuesField("val", i));
+ expected.put("d" + i, (long) i);
+ w.addDocument(d);
+ }
+ for (int u = 0; u < atLeast(200); u++) {
+ String id = "d" + random().nextInt(numDocs);
+ long v = random().nextLong();
+ w.updateNumericDocValue(new Term("id", id), "val", v);
+ expected.put(id, v);
+ if (rarely()) {
+ w.commit();
+ }
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (Map.Entry<String, Long> e : expected.entrySet()) {
+ assertNumeric(reader, e.getKey(), e.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * addIndexes(CodecReader...) flattens the source overlay into the destination, preserving the
+ * latest values.
+ */
+ public void testAddIndexesFromUpdatedIndex() throws Exception {
+ try (Directory src = newDirectory();
+ Directory dst = newDirectory()) {
+ Map<String, Long> expected = new HashMap<>();
+ try (IndexWriter w = new IndexWriter(src, incrementalConfig())) {
+ int numDocs = atLeast(30);
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ expected.put("d" + i, (long) i);
+ w.addDocument(d);
+ }
+ for (int u = 0; u < atLeast(150); u++) {
+ String id = "d" + random().nextInt(numDocs);
+ long v = random().nextLong();
+ w.updateNumericDocValue(new Term("id", id), "val", v);
+ expected.put(id, v);
+ }
+ w.commit();
+ }
+ try (IndexWriter w = new IndexWriter(dst, incrementalConfig());
+ DirectoryReader src2 = DirectoryReader.open(src)) {
+ CodecReader[] readers = new CodecReader[src2.leaves().size()];
+ for (int i = 0; i < readers.length; i++) {
+ readers[i] = (CodecReader) src2.leaves().get(i).reader();
+ }
+ w.addIndexes(readers);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (Map.Entry<String, Long> e : expected.entrySet()) {
+ assertNumeric(reader, e.getKey(), e.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ public void testBinarySetOnlyUpdates() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ int numDocs = atLeast(30);
+ Map<String, String> expected = new HashMap<>();
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new BinaryDocValuesField("val", new BytesRef("v" + i)));
+ expected.put("d" + i, "v" + i);
+ w.addDocument(d);
+ }
+ int updates = atLeast(120);
+ for (int i = 0; i < updates; i++) {
+ String id = "d" + random().nextInt(numDocs);
+ String v = "u" + random().nextInt(1_000_000);
+ w.updateBinaryDocValue(new Term("id", id), "val", new BytesRef(v));
+ expected.put(id, v);
+ if (rarely()) {
+ w.forceMerge(1);
+ }
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ for (Map.Entry<String, String> e : expected.entrySet()) {
+ boolean found = false;
+ for (LeafReaderContext ctx : reader.leaves()) {
+ TermsEnum te = ctx.reader().terms("id").iterator();
+ if (te.seekExact(new BytesRef(e.getKey()))) {
+ int doc = te.postings(null).nextDoc();
+ BinaryDocValues dv = ctx.reader().getBinaryDocValues("val");
+ assertTrue(e.getKey(), dv.advanceExact(doc));
+ assertEquals(e.getKey(), new BytesRef(e.getValue()), dv.binaryValue());
+ found = true;
+ break;
+ }
+ }
+ assertTrue("id not found: " + e.getKey(), found);
+ }
+ }
+ }
+ }
+
+ /**
+ * Doc-values updates on a skip-indexed field are rejected up front by {@link IndexWriter}. That
+ * pre-existing restriction is what lets the overlay assume a field's skipper always has a single
+ * producer (skippers are never overlaid), so the incremental path needs no special handling for
+ * them.
+ */
+ public void testCannotUpdateSkipIndexedField() throws Exception {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ Document d = new Document();
+ d.add(new StringField("id", "0", StringField.Store.NO));
+ d.add(NumericDocValuesField.indexedField("val", 1L));
+ w.addDocument(d);
+ w.commit();
+ IllegalArgumentException e =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> w.updateNumericDocValue(new Term("id", "0"), "val", 2L));
+ assertTrue(e.getMessage(), e.getMessage().contains("doc values skip index"));
+ }
+ }
+
+ /**
+ * A reader that predates the feature rejects the index (before any codec) rather than misreading
+ * a delta as the whole column, and the overlay round-trips through the segments file.
+ */
+ public void testOverlaySegmentRejectedByOlderReaders() throws Exception {
+ Directory dir = newDirectory();
+ try (IndexWriter w = new IndexWriter(dir, incrementalConfig())) {
+ Document d = new Document();
+ d.add(new StringField("id", "0", StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", 1L));
+ w.addDocument(d);
+ w.commit();
+ w.updateNumericDocValue(new Term("id", "0"), "val", 2L); // writes a sparse overlay generation
+ w.commit();
+ }
+ // The commit records overlay generations, so its segments file is written at VERSION_11_0; a
+ // reader that only understands up to VERSION_86 rejects it with IndexFormatTooNewException.
+ assertOldReaderRejects(dir);
+ // And the current reader sees the overlay round-tripped through the segments file.
+ assertTrue(
+ SegmentInfos.readLatestCommit(dir).asList().stream()
+ .anyMatch(SegmentCommitInfo::hasDocValuesOverlays));
+ dir.close();
+ }
+
+ /**
+ * After the deltas first fold back to a dense column, further updates must keep writing sparse
+ * deltas over that dense base rather than degrading to a full-column rewrite on every fold.
+ */
+ public void testSparseFoldOverDenseBase() throws Exception {
+ int numDocs = 10;
+ try (Directory dir = newDirectory();
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxDocValuesOverlays(1)
+ .setMergePolicy(NoMergePolicy.INSTANCE))) {
+ for (int i = 0; i < numDocs; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", "d" + i, StringField.Store.NO));
+ d.add(new NumericDocValuesField("val", i));
+ w.addDocument(d);
+ }
+ w.commit();
+ // Update every doc twice: the second round's coverage reaches maxDoc and folds to a dense
+ // column, so the field's base becomes a dense generation rather than the core column.
+ for (int round = 1; round <= 2; round++) {
+ for (int i = 0; i < numDocs; i++) {
+ w.updateNumericDocValue(new Term("id", "d" + i), "val", 100L * round + i);
+ }
+ w.commit();
+ }
+ // Now keep updating a single doc across two more folds over the dense base.
+ for (int round = 0; round < 2; round++) {
+ w.updateNumericDocValue(new Term("id", "d0"), "val", 999L + round);
+ w.commit();
+ }
+ // The last fold over the dense base must stay sparse: an overlay whose base is a dense
+ // generation (!= -1) with a folded delta, not a full-column rewrite that clears the overlay.
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
+ assertEquals(1, sis.size());
+ Map<Integer, long[]> overlays = sis.info(0).getDocValuesOverlays();
+ assertFalse("expected a sparse overlay over the dense base", overlays.isEmpty());
+ long[] packed = overlays.values().iterator().next();
+ assertTrue("base should be a dense generation, not the core column", packed[0] != -1);
+ assertTrue("expected at least one delta over the dense base", packed.length >= 2);
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ assertNumeric(reader, "d0", 1000L);
+ for (int i = 1; i < numDocs; i++) {
+ assertNumeric(reader, "d" + i, 200L + i);
+ }
+ }
+ }
+ }
+
+ private static void assertOldReaderRejects(Directory dir) throws IOException {
+ String segmentsFile = SegmentInfos.getLastCommitSegmentsFileName(dir);
+ try (ChecksumIndexInput in = dir.openChecksumInput(segmentsFile)) {
+ assertEquals(CodecUtil.CODEC_MAGIC, CodecUtil.readBEInt(in));
+ expectThrows(
+ IndexFormatTooNewException.class,
+ () ->
+ CodecUtil.checkHeaderNoMagic(
+ in, "segments", SegmentInfos.VERSION_74, SegmentInfos.VERSION_86));
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
index d5f014b302d..be2bbb6f911 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
@@ -1733,7 +1733,11 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
public void testDeleteUnusedUpdatesFiles() throws Exception {
Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ // Asserts the dense-rewrite behavior of superseding a field's prior generation, so it disables
+ // the sparse incremental path, which intentionally retains prior overlays to layer them at read
+ // time.
+ IndexWriterConfig conf =
+ newIndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(0);
IndexWriter writer = new IndexWriter(dir, conf);
Document doc = new Document();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOverlayBinaryDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestOverlayBinaryDocValues.java
new file mode 100644
index 00000000000..cbec7c50458
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOverlayBinaryDocValues.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
+
+public class TestOverlayBinaryDocValues extends LuceneTestCase {
+
+ private static BinaryDocValues sparse(int[] docs, String[] values) {
+ return new BinaryDocValues() {
+ int i = -1;
+ int doc = -1;
+
+ @Override
+ public BytesRef binaryValue() {
+ return new BytesRef(values[i]);
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int advance(int target) {
+ i++;
+ while (i < docs.length && docs[i] < target) {
+ i++;
+ }
+ doc = i < docs.length ? docs[i] : DocIdSetIterator.NO_MORE_DOCS;
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() {
+ return advance(doc + 1);
+ }
+
+ @Override
+ public boolean advanceExact(int target) {
+ int j = Math.max(i, 0);
+ while (j < docs.length && docs[j] < target) {
+ j++;
+ }
+ i = j;
+ doc = target;
+ return i < docs.length && docs[i] == target;
+ }
+
+ @Override
+ public long cost() {
+ return docs.length;
+ }
+ };
+ }
+
+ public void testAdvanceExactNewestWins() throws IOException {
+ BinaryDocValues base = sparse(new int[] {0, 1, 2, 3}, new String[] {"a0", "a1", "a2", "a3"});
+ BinaryDocValues d1 = sparse(new int[] {1, 2}, new String[] {"b1", "b2"});
+ BinaryDocValues d2 = sparse(new int[] {2}, new String[] {"c2"});
+ String[] expected = {"a0", "b1", "c2", "a3"};
+
+ OverlayBinaryDocValues overlay =
+ new OverlayBinaryDocValues(new BinaryDocValues[] {d2, d1, base});
+ for (int doc = 0; doc < 4; doc++) {
+ assertTrue("doc " + doc, overlay.advanceExact(doc));
+ assertEquals("doc " + doc, new BytesRef(expected[doc]), overlay.binaryValue());
+ }
+ }
+
+ public void testNextDocUnionMerge() throws IOException {
+ BinaryDocValues base = sparse(new int[] {0, 4}, new String[] {"a0", "a4"});
+ BinaryDocValues d1 =
+ sparse(new int[] {4, 6}, new String[] {"b4", "b6"}); // newest wins on doc 4
+ OverlayBinaryDocValues overlay = new OverlayBinaryDocValues(new BinaryDocValues[] {d1, base});
+
+ List<Integer> seen = new ArrayList<>();
+ List<String> vals = new ArrayList<>();
+ for (int doc = overlay.nextDoc();
+ doc != DocIdSetIterator.NO_MORE_DOCS;
+ doc = overlay.nextDoc()) {
+ seen.add(doc);
+ vals.add(overlay.binaryValue().utf8ToString());
+ }
+ assertEquals(List.of(0, 4, 6), seen);
+ assertEquals(List.of("a0", "b4", "b6"), vals);
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOverlayNumericDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestOverlayNumericDocValues.java
new file mode 100644
index 00000000000..9947de969fa
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOverlayNumericDocValues.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+public class TestOverlayNumericDocValues extends LuceneTestCase {
+
+ /**
+ * Sparse array-backed {@link NumericDocValues} over ascending docs; used in one access mode per
+ * instance.
+ */
+ private static NumericDocValues sparse(int[] docs, long[] values) {
+ return new NumericDocValues() {
+ int i = -1;
+ int doc = -1;
+
+ @Override
+ public long longValue() {
+ return values[i];
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int advance(int target) {
+ i++;
+ while (i < docs.length && docs[i] < target) {
+ i++;
+ }
+ doc = i < docs.length ? docs[i] : DocIdSetIterator.NO_MORE_DOCS;
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() {
+ return advance(doc + 1);
+ }
+
+ @Override
+ public boolean advanceExact(int target) {
+ int j = Math.max(i, 0);
+ while (j < docs.length && docs[j] < target) {
+ j++;
+ }
+ i = j;
+ doc = target;
+ return i < docs.length && docs[i] == target;
+ }
+
+ @Override
+ public long cost() {
+ return docs.length;
+ }
+ };
+ }
+
+ /** base dense 0..9, an older delta on {2,5}, a newest delta on {5,7}: newest wins on doc 5. */
+ public void testAdvanceExactNewestWins() throws IOException {
+ NumericDocValues base =
+ sparse(
+ new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+ new long[] {100, 101, 102, 103, 104, 105, 106, 107, 108, 109});
+ NumericDocValues d1 = sparse(new int[] {2, 5}, new long[] {201, 205});
+ NumericDocValues d2 = sparse(new int[] {5, 7}, new long[] {305, 307});
+ long[] expected = {100, 101, 201, 103, 104, 305, 106, 307, 108, 109};
+
+ OverlayNumericDocValues overlay =
+ new OverlayNumericDocValues(new NumericDocValues[] {d2, d1, base});
+ for (int doc = 0; doc < 10; doc++) {
+ assertTrue("doc " + doc, overlay.advanceExact(doc));
+ assertEquals("doc " + doc, expected[doc], overlay.longValue());
+ }
+ }
+
+ public void testNextDocUnionMerge() throws IOException {
+ NumericDocValues base =
+ sparse(
+ new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+ new long[] {100, 101, 102, 103, 104, 105, 106, 107, 108, 109});
+ NumericDocValues d1 = sparse(new int[] {2, 5}, new long[] {201, 205});
+ NumericDocValues d2 = sparse(new int[] {5, 7}, new long[] {305, 307});
+ long[] expected = {100, 101, 201, 103, 104, 305, 106, 307, 108, 109};
+
+ OverlayNumericDocValues overlay =
+ new OverlayNumericDocValues(new NumericDocValues[] {d2, d1, base});
+ List<Integer> seen = new ArrayList<>();
+ for (int doc = overlay.nextDoc();
+ doc != DocIdSetIterator.NO_MORE_DOCS;
+ doc = overlay.nextDoc()) {
+ seen.add(doc);
+ assertEquals("doc " + doc, expected[doc], overlay.longValue());
+ }
+ assertEquals(List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), seen);
+ }
+
+ /** Sparse base: docs not covered by any layer have no value; the union still merges correctly. */
+ public void testSparseBaseNextDocAndAdvanceExact() throws IOException {
+ NumericDocValues base = sparse(new int[] {0, 4}, new long[] {100, 104});
+ NumericDocValues d1 = sparse(new int[] {4, 6}, new long[] {204, 206}); // newest wins on doc 4
+ OverlayNumericDocValues overlay =
+ new OverlayNumericDocValues(new NumericDocValues[] {d1, base});
+
+ List<Integer> seen = new ArrayList<>();
+ for (int doc = overlay.nextDoc();
+ doc != DocIdSetIterator.NO_MORE_DOCS;
+ doc = overlay.nextDoc()) {
+ seen.add(doc);
+ }
+ assertEquals(List.of(0, 4, 6), seen);
+
+ OverlayNumericDocValues ra =
+ new OverlayNumericDocValues(
+ new NumericDocValues[] {
+ sparse(new int[] {4, 6}, new long[] {204, 206}),
+ sparse(new int[] {0, 4}, new long[] {100, 104})
+ });
+ assertTrue(ra.advanceExact(0));
+ assertEquals(100, ra.longValue());
+ assertFalse("doc 1 has no value", ra.advanceExact(1));
+ assertTrue(ra.advanceExact(4));
+ assertEquals(204, ra.longValue()); // newest layer wins
+ assertFalse("doc 5 has no value", ra.advanceExact(5));
+ assertTrue(ra.advanceExact(6));
+ assertEquals(206, ra.longValue());
+ }
+
+ public void testAdvanceSkips() throws IOException {
+ NumericDocValues base =
+ sparse(
+ new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+ new long[] {100, 101, 102, 103, 104, 105, 106, 107, 108, 109});
+ NumericDocValues d1 = sparse(new int[] {5}, new long[] {205});
+ OverlayNumericDocValues overlay =
+ new OverlayNumericDocValues(new NumericDocValues[] {d1, base});
+ assertEquals(5, overlay.advance(5));
+ assertEquals(205, overlay.longValue());
+ assertEquals(6, overlay.nextDoc());
+ assertEquals(106, overlay.longValue());
+ assertEquals(9, overlay.advance(9));
+ assertEquals(109, overlay.longValue());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, overlay.nextDoc());
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java b/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java
index 3c59c696763..2635be05bb9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java
@@ -46,7 +46,15 @@ public class TestReaderPool extends LuceneTestCase {
ReaderPool pool =
new ReaderPool(
- directory, directory, segmentInfos, fieldNumbers, () -> 0l, null, null, null);
+ directory,
+ directory,
+ segmentInfos,
+ fieldNumbers,
+ () -> 0l,
+ null,
+ null,
+ null,
+ new IndexWriterConfig());
SegmentCommitInfo commitInfo = RandomPicks.randomFrom(random(), segmentInfos.asList());
ReadersAndUpdates readersAndUpdates = pool.get(commitInfo, true);
assertSame(readersAndUpdates, pool.get(commitInfo, false));
@@ -67,7 +75,15 @@ public class TestReaderPool extends LuceneTestCase {
ReaderPool pool =
new ReaderPool(
- directory, directory, segmentInfos, fieldNumbers, () -> 0l, null, null, null);
+ directory,
+ directory,
+ segmentInfos,
+ fieldNumbers,
+ () -> 0l,
+ null,
+ null,
+ null,
+ new IndexWriterConfig());
SegmentCommitInfo commitInfo = RandomPicks.randomFrom(random(), segmentInfos.asList());
assertFalse(pool.isReaderPoolingEnabled());
pool.release(pool.get(commitInfo, true), random().nextBoolean());
@@ -111,7 +127,8 @@ public class TestReaderPool extends LuceneTestCase {
() -> 0l,
new NullInfoStream(),
null,
- null);
+ null,
+ new IndexWriterConfig());
int id = random().nextInt(10);
if (random().nextBoolean()) {
pool.enableReaderPooling();
@@ -188,7 +205,8 @@ public class TestReaderPool extends LuceneTestCase {
() -> 0l,
new NullInfoStream(),
null,
- null);
+ null,
+ new IndexWriterConfig());
int id = random().nextInt(10);
if (random().nextBoolean()) {
pool.enableReaderPooling();
@@ -241,7 +259,8 @@ public class TestReaderPool extends LuceneTestCase {
() -> 0L,
new NullInfoStream(),
null,
- null);
+ null,
+ new IndexWriterConfig());
if (random().nextBoolean()) {
pool.enableReaderPooling();
}
@@ -329,7 +348,8 @@ public class TestReaderPool extends LuceneTestCase {
() -> 0l,
new NullInfoStream(),
null,
- null);
+ null,
+ new IndexWriterConfig());
assertEquals(0, pool.getReadersByRam().size());
int ord = 0;
diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java b/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
index 7be17ae784c..86bd45a07fe 100644
--- a/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
+++ b/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java
@@ -350,8 +350,10 @@ public final class IndexUtils {
format = "Lucene 7.4 or later";
} else if (actualVersion == SegmentInfos.VERSION_86) {
format = "Lucene 8.6 or later";
- } else if (actualVersion > SegmentInfos.VERSION_86) {
- format = "Lucene 8.6 or later (UNSUPPORTED)";
+ } else if (actualVersion == SegmentInfos.VERSION_11_0) {
+ format = "Lucene 11.0 or later";
+ } else if (actualVersion > SegmentInfos.VERSION_11_0) {
+ format = "Lucene 11.0 or later (UNSUPPORTED)";
}
} else {
format = "Lucene 6.x or prior (UNSUPPORTED)";
diff --git a/lucene/luke/src/test/org/apache/lucene/luke/models/overview/TestOverviewImpl.java b/lucene/luke/src/test/org/apache/lucene/luke/models/overview/TestOverviewImpl.java
index dc954aeeb11..24b0a065088 100644
--- a/lucene/luke/src/test/org/apache/lucene/luke/models/overview/TestOverviewImpl.java
+++ b/lucene/luke/src/test/org/apache/lucene/luke/models/overview/TestOverviewImpl.java
@@ -73,7 +73,7 @@ public class TestOverviewImpl extends OverviewTestBase {
public void testGetIndexFormat() {
OverviewImpl overview = new OverviewImpl(reader, indexDir.toString());
- assertEquals("Lucene 8.6 or later", overview.getIndexFormat().get());
+ assertEquals("Lucene 11.0 or later", overview.getIndexFormat().get());
}
public void testGetDirImpl() {