You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2026/06/19 08:30:49 UTC
(lucene) branch main updated: Check if merge is aborted before executing file integrity checks (#16264)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new a34cefd33cb Check if merge is aborted before executing file integrity checks (#16264)
a34cefd33cb is described below
commit a34cefd33cbd5eeb9e6c01de60cada745fa45149
Author: Tanguy Leroux <tl...@gmail.com>
AuthorDate: Fri Jun 19 10:30:43 2026 +0200
Check if merge is aborted before executing file integrity checks (#16264)
Adds OneMerge to the MergeState so that merge writers and consumers
can check the abort flag before starting each reader's checkIntegrity().
This avoids kicking off costly full-file checksums on segments when the
merge has already been aborted.
---
lucene/CHANGES.txt | 3 +++
.../apache/lucene/codecs/DocValuesConsumer.java | 1 +
.../org/apache/lucene/codecs/FieldsConsumer.java | 1 +
.../org/apache/lucene/codecs/KnnVectorsWriter.java | 1 +
.../org/apache/lucene/codecs/NormsConsumer.java | 1 +
.../org/apache/lucene/codecs/PointsWriter.java | 1 +
.../apache/lucene/codecs/StoredFieldsWriter.java | 1 +
.../apache/lucene/codecs/TermVectorsWriter.java | 1 +
.../codecs/lucene90/Lucene90PointsWriter.java | 1 +
.../Lucene90CompressingStoredFieldsWriter.java | 1 +
.../Lucene90CompressingTermVectorsWriter.java | 1 +
.../lucene/codecs/perfield/PerFieldMergeState.java | 3 ++-
.../java/org/apache/lucene/index/IndexWriter.java | 6 +++--
.../java/org/apache/lucene/index/MergeState.java | 26 ++++++++++++++++++++--
.../org/apache/lucene/index/SegmentMerger.java | 6 +++--
.../lucene/codecs/TestMergedVectorValues.java | 4 ++--
.../src/test/org/apache/lucene/index/TestDoc.java | 3 ++-
.../org/apache/lucene/index/TestSegmentMerger.java | 3 ++-
18 files changed, 53 insertions(+), 11 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ad7b71d0e74..fa21530f244 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -410,6 +410,9 @@ Improvements
DiversifiedTopDocsCollector, removing usages of the deprecated IndexSearcher#search(Query, Collector).
(Luca Cavanna)
+* GITHUB#16264: Check if merge is aborted before executing file integrity checks to avoid
+ costly full-file checksums on segments when the merge has already been cancelled. (Tanguy Leroux)
+
Optimizations
---------------------
* GITHUB#16111: Optimize FieldExistsQuery.count() when all docs have the field. (Prithvi S)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
index 0c26af89152..b75cce9b1a5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
@@ -132,6 +132,7 @@ public abstract class DocValuesConsumer implements Closeable {
public void merge(MergeState mergeState) throws IOException {
for (DocValuesProducer docValuesProducer : mergeState.docValuesProducers) {
if (docValuesProducer != null) {
+ mergeState.checkAborted();
docValuesProducer.checkIntegrity();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java
index 5443585ab97..537410f8569 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java
@@ -80,6 +80,7 @@ public abstract class FieldsConsumer implements Closeable {
final int maxDoc = mergeState.maxDocs[readerIndex];
if (f != null) {
+ mergeState.checkAborted();
f.checkIntegrity();
slices.add(new ReaderSlice(docBase, maxDoc, readerIndex));
fields.add(f);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java
index d2215e99808..d1e2b2ee1c4 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java
@@ -113,6 +113,7 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {
KnnVectorsReader reader = mergeState.knnVectorsReaders[i];
assert reader != null || mergeState.fieldInfos[i].hasVectorValues() == false;
if (reader != null) {
+ mergeState.checkAborted();
reader.checkIntegrity();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java
index 7147e6b3327..fedd9584db3 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java
@@ -66,6 +66,7 @@ public abstract class NormsConsumer implements Closeable {
public void merge(MergeState mergeState) throws IOException {
for (NormsProducer normsProducer : mergeState.normsProducers) {
if (normsProducer != null) {
+ mergeState.checkAborted();
normsProducer.checkIntegrity();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
index 9d756e36d6f..4b4f2a01773 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java
@@ -222,6 +222,7 @@ public abstract class PointsWriter implements Closeable {
// check each incoming reader
for (PointsReader reader : mergeState.pointsReaders) {
if (reader != null) {
+ mergeState.checkAborted();
reader.checkIntegrity();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
index a9f9ce464a3..f5eb9e4c113 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
@@ -130,6 +130,7 @@ public abstract class StoredFieldsWriter implements Closeable, Accountable {
List<StoredFieldsMergeSub> subs = new ArrayList<>();
for (int i = 0; i < mergeState.storedFieldsReaders.length; i++) {
StoredFieldsReader storedFieldsReader = mergeState.storedFieldsReaders[i];
+ mergeState.checkAborted();
storedFieldsReader.checkIntegrity();
subs.add(
new StoredFieldsMergeSub(
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
index 45dcc504186..8c9b11dd352 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
@@ -200,6 +200,7 @@ public abstract class TermVectorsWriter implements Closeable, Accountable {
for (int i = 0; i < mergeState.termVectorsReaders.length; i++) {
TermVectorsReader reader = mergeState.termVectorsReaders[i];
if (reader != null) {
+ mergeState.checkAborted();
reader.checkIntegrity();
}
subs.add(new TermVectorsMergeSub(mergeState.docMaps[i], reader, mergeState.maxDocs[i]));
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java
index 7f56d855473..eaea9d45bf5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java
@@ -204,6 +204,7 @@ public class Lucene90PointsWriter extends PointsWriter {
}
for (PointsReader reader : mergeState.pointsReaders) {
if (reader != null) {
+ mergeState.checkAborted();
reader.checkIntegrity();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
index 39009b1eb56..4d2b2f25a14 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
@@ -601,6 +601,7 @@ public final class Lucene90CompressingStoredFieldsWriter extends StoredFieldsWri
new ArrayList<>(mergeState.storedFieldsReaders.length);
for (int i = 0; i < mergeState.storedFieldsReaders.length; i++) {
final StoredFieldsReader reader = mergeState.storedFieldsReaders[i];
+ mergeState.checkAborted();
reader.checkIntegrity();
MergeStrategy mergeStrategy = getMergeStrategy(mergeState, matchingReaders, i);
if (mergeStrategy == MergeStrategy.VISITOR) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java
index 00eddc03ee8..3fe9f2a47ef 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java
@@ -898,6 +898,7 @@ public final class Lucene90CompressingTermVectorsWriter extends TermVectorsWrite
for (int i = 0; i < numReaders; i++) {
final TermVectorsReader reader = mergeState.termVectorsReaders[i];
if (reader != null) {
+ mergeState.checkAborted();
reader.checkIntegrity();
}
final boolean bulkMerge = canPerformBulkMerge(mergeState, matchingReaders, i);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
index bb67ec618fd..85df2414aa3 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java
@@ -69,7 +69,8 @@ final class PerFieldMergeState {
in.maxDocs,
in.infoStream,
in.intraMergeTaskExecutor,
- in.needsIndexSort);
+ in.needsIndexSort,
+ in.oneMerge);
}
private static class FilterFieldInfos extends FieldInfos {
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index bf806dd08c0..68ddd1611b9 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -3521,7 +3521,8 @@ public class IndexWriter
trackingDir,
globalFieldNumberMap,
context,
- intraMergeExecutor);
+ intraMergeExecutor,
+ merge);
try {
if (!merger.shouldMerge()) {
return;
@@ -5322,7 +5323,8 @@ public class IndexWriter
dirWrapper,
globalFieldNumberMap,
context,
- intraMergeExecutor);
+ intraMergeExecutor,
+ merge);
MergeState mergeState = merger.mergeState;
MergeState.DocMap[] docMaps;
try {
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeState.java b/lucene/core/src/java/org/apache/lucene/index/MergeState.java
index 838699215f0..fafdafbb0a4 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergeState.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergeState.java
@@ -91,14 +91,24 @@ public class MergeState {
/** Indicates if the index needs to be sorted * */
public boolean needsIndexSort;
+ /**
+ * The merge that this state is associated with, or {@code null} if this merge state is not
+ * associated with an {@link IndexWriter} merge (e.g. for addIndexes).
+ *
+ * @lucene.internal
+ */
+ public final MergePolicy.OneMerge oneMerge;
+
/** Sole constructor. */
MergeState(
List<CodecReader> readers,
SegmentInfo segmentInfo,
InfoStream infoStream,
- Executor intraMergeTaskExecutor)
+ Executor intraMergeTaskExecutor,
+ MergePolicy.OneMerge oneMerge)
throws IOException {
verifyIndexSort(readers, segmentInfo);
+ this.oneMerge = oneMerge;
this.infoStream = infoStream;
int numReaders = readers.size();
this.intraMergeTaskExecutor = intraMergeTaskExecutor;
@@ -230,6 +240,16 @@ public class MergeState {
}
}
+ /**
+ * Checks if the merge has been aborted, throwing {@link MergePolicy.MergeAbortedException} if so.
+ * This is a no-op if this merge state is not associated with an {@link IndexWriter} merge.
+ */
+ public void checkAborted() throws MergePolicy.MergeAbortedException {
+ if (oneMerge != null) {
+ oneMerge.checkAborted();
+ }
+ }
+
private static void verifyIndexSort(List<CodecReader> readers, SegmentInfo segmentInfo) {
Sort indexSort = segmentInfo.getIndexSort();
if (indexSort == null) {
@@ -284,7 +304,8 @@ public class MergeState {
int[] maxDocs,
InfoStream infoStream,
Executor intraMergeTaskExecutor,
- boolean needsIndexSort) {
+ boolean needsIndexSort,
+ MergePolicy.OneMerge oneMerge) {
this.docMaps = docMaps;
this.segmentInfo = segmentInfo;
this.mergeFieldInfos = mergeFieldInfos;
@@ -301,5 +322,6 @@ public class MergeState {
this.infoStream = infoStream;
this.intraMergeTaskExecutor = intraMergeTaskExecutor;
this.needsIndexSort = needsIndexSort;
+ this.oneMerge = oneMerge;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
index 8dab44c1c8e..321780406ad 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
@@ -60,18 +60,20 @@ final class SegmentMerger {
Directory dir,
FieldInfos.FieldNumbers fieldNumbers,
IOContext context,
- Executor intraMergeTaskExecutor)
+ Executor intraMergeTaskExecutor,
+ MergePolicy.OneMerge oneMerge)
throws IOException {
if (context.context() != IOContext.Context.MERGE) {
throw new IllegalArgumentException(
"IOContext.context should be MERGE; got: " + context.context());
}
- mergeState = new MergeState(readers, segmentInfo, infoStream, intraMergeTaskExecutor);
mergeStateCreationThread = Thread.currentThread();
directory = dir;
this.codec = segmentInfo.getCodec();
this.context = context;
this.fieldInfosBuilder = new FieldInfos.Builder(fieldNumbers);
+ this.mergeState =
+ new MergeState(readers, segmentInfo, infoStream, intraMergeTaskExecutor, oneMerge);
Version minVersion = Version.LATEST;
for (CodecReader reader : readers) {
Version leafMinVersion = reader.getMetaData().minVersion();
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java b/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java
index 5f1fc6a2267..e00a1d8c96a 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java
@@ -40,7 +40,7 @@ public class TestMergedVectorValues extends LuceneTestCase {
MergeState state =
new MergeState(
null, null, null, null, null, null, null, null, null, null, null, null, null, null,
- null, false);
+ null, false, null);
// Run the test
ByteVectorValues values =
@@ -68,7 +68,7 @@ public class TestMergedVectorValues extends LuceneTestCase {
MergeState state =
new MergeState(
null, null, null, null, null, null, null, null, null, null, null, null, null, null,
- null, false);
+ null, false, null);
// Run the test
FloatVectorValues values =
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
index 13384fcb892..2151b24157c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
@@ -239,7 +239,8 @@ public class TestDoc extends LuceneTestCase {
trackingDir,
new FieldInfos.FieldNumbers(null, null),
context,
- new SameThreadExecutorService());
+ new SameThreadExecutorService(),
+ null);
merger.merge();
merger.cleanupMerge();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
index 762e934a636..56dd6ed2e2e 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
@@ -107,7 +107,8 @@ public class TestSegmentMerger extends LuceneTestCase {
mergedDir,
new FieldInfos.FieldNumbers(null, null),
newIOContext(random(), IOContext.merge(new MergeInfo(-1, -1, false, -1))),
- new SameThreadExecutorService());
+ new SameThreadExecutorService(),
+ null);
MergeState mergeState = merger.merge();
merger.cleanupMerge();
int docsMerged = mergeState.segmentInfo.maxDoc();