You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by vi...@apache.org on 2022/08/01 12:43:50 UTC
[lucene] branch branch_9x updated: LUCENE-10216: Use MergeScheduler and MergePolicy to run addIndexes(CodecReader[]) merges. (#1051)
This is an automated email from the ASF dual-hosted git repository.
vigyasharma pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 5dd8e9bdc5a LUCENE-10216: Use MergeScheduler and MergePolicy to run addIndexes(CodecReader[]) merges. (#1051)
5dd8e9bdc5a is described below
commit 5dd8e9bdc5ae72fc726a98a64bfce5119c77b558
Author: Vigya Sharma <vi...@gmail.com>
AuthorDate: Mon Aug 1 05:43:46 2022 -0700
LUCENE-10216: Use MergeScheduler and MergePolicy to run addIndexes(CodecReader[]) merges. (#1051)
Use merge policy and merge scheduler to run addIndexes merges.
This is a back port of the following commits from main:
* LUCENE-10216: Use MergeScheduler and MergePolicy to run addIndexes(CodecReader[]) merges. (#633)
* LUCENE-10648: Fix failures in TestAssertingPointsFormat.testWithExceptions (#1012)
---
lucene/CHANGES.txt | 3 +
.../java/org/apache/lucene/index/FieldInfos.java | 8 +
.../org/apache/lucene/index/FilterMergePolicy.java | 5 +
.../java/org/apache/lucene/index/IndexWriter.java | 423 +++++++++++++++------
.../java/org/apache/lucene/index/MergePolicy.java | 82 +++-
.../java/org/apache/lucene/index/MergeTrigger.java | 2 +
.../org/apache/lucene/index/NoMergePolicy.java | 10 +
.../org/apache/lucene/index/TestAddIndexes.java | 304 ++++++++++++++-
.../lucene/index/TestIndexWriterOnDiskFull.java | 4 +-
9 files changed, 704 insertions(+), 137 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index af67906dd2f..776309b80ec 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -17,6 +17,9 @@ Improvements
---------------------
* LUCENE-10592: Build HNSW Graph on indexing. (Mayya Sharipova, Adrien Grand, Julie Tibshirani)
+* LUCENE-10216: Use MergePolicy to define and MergeScheduler to trigger the reader merges
+ required by addIndexes(CodecReader[]) API. (Vigya Sharma, Michael McCandless)
+
Optimizations
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
index 1df0f679375..39c118f833c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
@@ -380,6 +380,14 @@ public class FieldInfos implements Iterable<FieldInfo> {
this.strictlyConsistent = indexCreatedVersionMajor >= 9;
}
+ void verifyFieldInfo(FieldInfo fi) {
+ String fieldName = fi.getName();
+ verifySoftDeletedFieldName(fieldName, fi.isSoftDeletesField());
+ if (nameToNumber.containsKey(fieldName)) {
+ verifySameSchema(fi);
+ }
+ }
+
/**
* Returns the global field number for the given field name. If the name does not exist yet it
* tries to add it with the given preferred field number assigned if possible otherwise the
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/FilterMergePolicy.java
index 6eeb61cde73..08c0196a413 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterMergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterMergePolicy.java
@@ -47,6 +47,11 @@ public class FilterMergePolicy extends MergePolicy implements Unwrappable<MergeP
return in.findMerges(mergeTrigger, segmentInfos, mergeContext);
}
+ @Override
+ public MergeSpecification findMerges(CodecReader... readers) throws IOException {
+ return in.findMerges(readers);
+ }
+
@Override
public MergeSpecification findForcedMerges(
SegmentInfos segmentInfos,
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index ecc2421ea87..1c437844906 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -276,6 +276,7 @@ public class IndexWriter
final DocumentsWriter docWriter;
private final EventQueue eventQueue = new EventQueue(this);
private final MergeScheduler.MergeSource mergeSource = new IndexWriterMergeSource(this);
+ private final AddIndexesMergeSource addIndexesMergeSource = new AddIndexesMergeSource(this);
private final ReentrantLock writeDocValuesLock = new ReentrantLock();
@@ -2336,6 +2337,10 @@ public class IndexWriter
case COMMIT:
spec = mergePolicy.findFullFlushMerges(trigger, segmentInfos, this);
break;
+ case ADD_INDEXES:
+ throw new IllegalStateException(
+ "Merges with ADD_INDEXES trigger should be "
+ + "called from within the addIndexes() API flow");
case EXPLICIT:
case FULL_FLUSH:
case MERGE_FINISHED:
@@ -2672,6 +2677,9 @@ public class IndexWriter
});
pendingMerges.clear();
+ // abort any merges pending from addIndexes(CodecReader...)
+ addIndexesMergeSource.abortPendingMerges();
+
for (final MergePolicy.OneMerge merge : runningMerges) {
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now abort running merge " + segString(merge.segments));
@@ -3116,13 +3124,10 @@ public class IndexWriter
*
* <p><b>NOTE:</b> empty segments are dropped by this method and not added to this index.
*
- * <p><b>NOTE:</b> this merges all given {@link LeafReader}s in one merge. If you intend to merge
- * a large number of readers, it may be better to call this method multiple times, each time with
- * a small set of readers. In principle, if you use a merge policy with a {@code mergeFactor} or
- * {@code maxMergeAtOnce} parameter, you should pass that many readers in one call.
- *
- * <p><b>NOTE:</b> this method does not call or make use of the {@link MergeScheduler}, so any
- * custom bandwidth throttling is at the moment ignored.
+ * <p><b>NOTE:</b> provided {@link LeafReader}s are merged as specified by the {@link
+ * MergePolicy#findMerges(CodecReader...)} API. Default behavior is to merge all provided readers
+ * into a single segment. You can modify this by overriding the <code>findMerge</code> API in your
+ * custom merge policy.
*
* @return The <a href="#sequence_number">sequence number</a> for this operation
* @throws CorruptIndexException if the index is corrupt
@@ -3136,142 +3141,310 @@ public class IndexWriter
// long so we can detect int overflow:
long numDocs = 0;
long seqNo;
- try {
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "flush at addIndexes(CodecReader...)");
- }
- flush(false, true);
- String mergedName = newSegmentName();
- int numSoftDeleted = 0;
+ try {
+ // Best effort up front validations
for (CodecReader leaf : readers) {
- numDocs += leaf.numDocs();
validateMergeReader(leaf);
- if (softDeletesEnabled) {
- Bits liveDocs = leaf.getLiveDocs();
- numSoftDeleted +=
- PendingSoftDeletes.countSoftDeletes(
- FieldExistsQuery.getDocValuesDocIdSetIterator(config.getSoftDeletesField(), leaf),
- liveDocs);
+ for (FieldInfo fi : leaf.getFieldInfos()) {
+ globalFieldNumberMap.verifyFieldInfo(fi);
}
+ numDocs += leaf.numDocs();
}
-
- // Best-effort up front check:
testReserveDocs(numDocs);
- final IOContext context =
- new IOContext(
- new MergeInfo(Math.toIntExact(numDocs), -1, false, UNBOUNDED_MAX_MERGE_SEGMENTS));
-
- // TODO: somehow we should fix this merge so it's
- // abortable so that IW.close(false) is able to stop it
- TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
- Codec codec = config.getCodec();
- // We set the min version to null for now, it will be set later by SegmentMerger
- SegmentInfo info =
- new SegmentInfo(
- directoryOrig,
- Version.LATEST,
- null,
- mergedName,
- -1,
- false,
- codec,
- Collections.emptyMap(),
- StringHelper.randomId(),
- Collections.emptyMap(),
- config.getIndexSort());
-
- SegmentMerger merger =
- new SegmentMerger(
- Arrays.asList(readers), info, infoStream, trackingDir, globalFieldNumberMap, context);
+ synchronized (this) {
+ ensureOpen();
+ if (merges.areEnabled() == false) {
+ throw new AlreadyClosedException(
+ "this IndexWriter is closed. Cannot execute addIndexes(CodecReaders...) API");
+ }
+ }
- if (!merger.shouldMerge()) {
+ MergePolicy mergePolicy = config.getMergePolicy();
+ MergePolicy.MergeSpecification spec = mergePolicy.findMerges(readers);
+ boolean mergeSuccess = false;
+ if (spec != null && spec.merges.size() > 0) {
+ try {
+ spec.merges.forEach(addIndexesMergeSource::registerMerge);
+ mergeScheduler.merge(addIndexesMergeSource, MergeTrigger.ADD_INDEXES);
+ spec.await();
+ mergeSuccess =
+ spec.merges.stream().allMatch(m -> m.hasCompletedSuccessfully().orElse(false));
+ } finally {
+ if (mergeSuccess == false) {
+ for (MergePolicy.OneMerge merge : spec.merges) {
+ if (merge.getMergeInfo() != null) {
+ deleteNewFiles(merge.getMergeInfo().files());
+ }
+ }
+ }
+ }
+ } else {
+ if (infoStream.isEnabled("IW")) {
+ if (spec == null) {
+ infoStream.message(
+ "addIndexes(CodecReaders...)",
+ "received null mergeSpecification from MergePolicy. No indexes to add, returning..");
+ } else {
+ infoStream.message(
+ "addIndexes(CodecReaders...)",
+ "received empty mergeSpecification from MergePolicy. No indexes to add, returning..");
+ }
+ }
return docWriter.getNextSequenceNumber();
}
- synchronized (this) {
- ensureOpen();
- assert merges.areEnabled();
- runningAddIndexesMerges.add(merger);
- }
- try {
- merger.merge(); // merge 'em
- } finally {
+ if (mergeSuccess) {
+ List<SegmentCommitInfo> infos = new ArrayList<>();
+ long totalDocs = 0;
+ for (MergePolicy.OneMerge merge : spec.merges) {
+ totalDocs += merge.totalMaxDoc;
+ if (merge.getMergeInfo() != null) {
+ infos.add(merge.getMergeInfo());
+ }
+ }
+
synchronized (this) {
- runningAddIndexesMerges.remove(merger);
- notifyAll();
+ if (infos.isEmpty() == false) {
+ boolean registerSegmentSuccess = false;
+ try {
+ ensureOpen();
+ // Reserve the docs, just before we update SIS:
+ reserveDocs(totalDocs);
+ registerSegmentSuccess = true;
+ } finally {
+ if (registerSegmentSuccess == false) {
+ for (SegmentCommitInfo sipc : infos) {
+ // Safe: these files must exist
+ deleteNewFiles(sipc.files());
+ }
+ }
+ }
+ segmentInfos.addAll(infos);
+ checkpoint();
+ }
+ seqNo = docWriter.getNextSequenceNumber();
}
+ } else {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message(
+ "addIndexes(CodecReaders...)", "failed to successfully merge all provided readers.");
+ }
+ for (MergePolicy.OneMerge merge : spec.merges) {
+ if (merge.isAborted()) {
+ throw new MergePolicy.MergeAbortedException("merge was aborted.");
+ }
+ Throwable t = merge.getException();
+ if (t != null) {
+ IOUtils.rethrowAlways(t);
+ }
+ }
+ // If no merge hit an exception, and merge was not aborted, but we still failed to add
+ // indexes, fail the API
+ throw new RuntimeException(
+ "failed to successfully merge all provided readers in addIndexes(CodecReader...)");
}
- SegmentCommitInfo infoPerCommit =
- new SegmentCommitInfo(info, 0, numSoftDeleted, -1L, -1L, -1L, StringHelper.randomId());
+ } catch (VirtualMachineError tragedy) {
+ tragicEvent(tragedy, "addIndexes(CodecReader...)");
+ throw tragedy;
+ }
- info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
- trackingDir.clearCreatedFiles();
+ maybeMerge();
+ return seqNo;
+ }
- setDiagnostics(info, SOURCE_ADDINDEXES_READERS);
+ private class AddIndexesMergeSource implements MergeScheduler.MergeSource {
- final MergePolicy mergePolicy = config.getMergePolicy();
- boolean useCompoundFile;
- synchronized (this) { // Guard segmentInfos
- if (merges.areEnabled() == false) {
- // Safe: these files must exist
- deleteNewFiles(infoPerCommit.files());
+ private final Queue<MergePolicy.OneMerge> pendingAddIndexesMerges = new ArrayDeque<>();
+ private final IndexWriter writer;
- return docWriter.getNextSequenceNumber();
- }
- ensureOpen();
- useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, infoPerCommit, this);
+ public AddIndexesMergeSource(IndexWriter writer) {
+ this.writer = writer;
+ }
+
+ public void registerMerge(MergePolicy.OneMerge merge) {
+ synchronized (IndexWriter.this) {
+ pendingAddIndexesMerges.add(merge);
}
+ }
- // Now create the compound file if needed
- if (useCompoundFile) {
- Collection<String> filesToDelete = infoPerCommit.files();
- TrackingDirectoryWrapper trackingCFSDir = new TrackingDirectoryWrapper(directory);
- // TODO: unlike merge, on exception we arent sniping any trash cfs files here?
- // createCompoundFile tries to cleanup, but it might not always be able to...
- try {
- createCompoundFile(infoStream, trackingCFSDir, info, context, this::deleteNewFiles);
- } finally {
- // delete new non cfs files directly: they were never
- // registered with IFD
- deleteNewFiles(filesToDelete);
+ @Override
+ public MergePolicy.OneMerge getNextMerge() {
+ synchronized (IndexWriter.this) {
+ if (hasPendingMerges() == false) {
+ return null;
}
- info.setUseCompoundFile(true);
+ MergePolicy.OneMerge merge = pendingAddIndexesMerges.remove();
+ runningMerges.add(merge);
+ return merge;
}
+ }
- // Have codec write SegmentInfo. Must do this after
- // creating CFS so that 1) .si isn't slurped into CFS,
- // and 2) .si reflects useCompoundFile=true change
- // above:
- codec.segmentInfoFormat().write(trackingDir, info, context);
+ @Override
+ public void onMergeFinished(MergePolicy.OneMerge merge) {
+ synchronized (IndexWriter.this) {
+ runningMerges.remove(merge);
+ }
+ }
- info.addFiles(trackingDir.getCreatedFiles());
+ @Override
+ public boolean hasPendingMerges() {
+ return pendingAddIndexesMerges.size() > 0;
+ }
- // Register the new segment
- synchronized (this) {
- if (merges.areEnabled() == false) {
- // Safe: these files must exist
- deleteNewFiles(infoPerCommit.files());
+ public void abortPendingMerges() throws IOException {
+ synchronized (IndexWriter.this) {
+ IOUtils.applyToAll(
+ pendingAddIndexesMerges,
+ merge -> {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "now abort pending addIndexes merge");
+ }
+ merge.setAborted();
+ merge.close(false, false, mr -> {});
+ onMergeFinished(merge);
+ });
+ pendingAddIndexesMerges.clear();
+ }
+ }
- return docWriter.getNextSequenceNumber();
+ @Override
+ public void merge(MergePolicy.OneMerge merge) throws IOException {
+ boolean success = false;
+ try {
+ writer.addIndexesReaderMerge(merge);
+ success = true;
+ } catch (Throwable t) {
+ handleMergeException(t, merge);
+ } finally {
+ synchronized (IndexWriter.this) {
+ merge.close(success, false, mr -> {});
+ onMergeFinished(merge);
}
- ensureOpen();
+ }
+ }
+ }
+
+ /**
+ * Runs a single merge operation for {@link IndexWriter#addIndexes(CodecReader...)}.
+ *
+ * <p>Merges and creates a SegmentInfo, for the readers grouped together in provided OneMerge.
+ *
+ * @param merge OneMerge object initialized from readers.
+ * @throws IOException if there is a low-level IO error
+ */
+ public void addIndexesReaderMerge(MergePolicy.OneMerge merge) throws IOException {
- // Now reserve the docs, just before we update SIS:
- reserveDocs(numDocs);
+ merge.mergeInit();
+ merge.checkAborted();
- segmentInfos.add(infoPerCommit);
- seqNo = docWriter.getNextSequenceNumber();
- checkpoint();
+ // long so we can detect int overflow:
+ long numDocs = 0;
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "flush at addIndexes(CodecReader...)");
+ }
+ flush(false, true);
+
+ String mergedName = newSegmentName();
+ Directory mergeDirectory = mergeScheduler.wrapForMerge(merge, directory);
+ int numSoftDeleted = 0;
+ for (MergePolicy.MergeReader reader : merge.getMergeReader()) {
+ CodecReader leaf = reader.codecReader;
+ numDocs += leaf.numDocs();
+ if (softDeletesEnabled) {
+ Bits liveDocs = reader.hardLiveDocs;
+ numSoftDeleted +=
+ PendingSoftDeletes.countSoftDeletes(
+ FieldExistsQuery.getDocValuesDocIdSetIterator(config.getSoftDeletesField(), leaf),
+ liveDocs);
}
- } catch (VirtualMachineError tragedy) {
- tragicEvent(tragedy, "addIndexes(CodecReader...)");
- throw tragedy;
}
- maybeMerge();
- return seqNo;
+ // Best-effort up front check:
+ testReserveDocs(numDocs);
+
+ final IOContext context =
+ new IOContext(
+ new MergeInfo(Math.toIntExact(numDocs), -1, false, UNBOUNDED_MAX_MERGE_SEGMENTS));
+
+ TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(mergeDirectory);
+ Codec codec = config.getCodec();
+ // We set the min version to null for now, it will be set later by SegmentMerger
+ SegmentInfo segInfo =
+ new SegmentInfo(
+ directoryOrig,
+ Version.LATEST,
+ null,
+ mergedName,
+ -1,
+ false,
+ codec,
+ Collections.emptyMap(),
+ StringHelper.randomId(),
+ Collections.emptyMap(),
+ config.getIndexSort());
+
+ List<CodecReader> readers =
+ merge.getMergeReader().stream().map(r -> r.codecReader).collect(Collectors.toList());
+ SegmentMerger merger =
+ new SegmentMerger(readers, segInfo, infoStream, trackingDir, globalFieldNumberMap, context);
+
+ if (!merger.shouldMerge()) {
+ return;
+ }
+
+ merge.checkAborted();
+ synchronized (this) {
+ runningAddIndexesMerges.add(merger);
+ }
+ merge.mergeStartNS = System.nanoTime();
+ try {
+ merger.merge(); // merge 'em
+ } finally {
+ synchronized (this) {
+ runningAddIndexesMerges.remove(merger);
+ notifyAll();
+ }
+ }
+
+ merge.setMergeInfo(
+ new SegmentCommitInfo(segInfo, 0, numSoftDeleted, -1L, -1L, -1L, StringHelper.randomId()));
+ merge.getMergeInfo().info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
+ trackingDir.clearCreatedFiles();
+
+ setDiagnostics(merge.getMergeInfo().info, SOURCE_ADDINDEXES_READERS);
+
+ final MergePolicy mergePolicy = config.getMergePolicy();
+ boolean useCompoundFile;
+ synchronized (this) {
+ merge.checkAborted();
+ useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, merge.getMergeInfo(), this);
+ }
+
+ // Now create the compound file if needed
+ if (useCompoundFile) {
+ Collection<String> filesToDelete = merge.getMergeInfo().files();
+ TrackingDirectoryWrapper trackingCFSDir = new TrackingDirectoryWrapper(mergeDirectory);
+ // createCompoundFile tries to cleanup, but it might not always be able to...
+ createCompoundFile(
+ infoStream, trackingCFSDir, merge.getMergeInfo().info, context, this::deleteNewFiles);
+
+ // creating cfs resets the files tracked in SegmentInfo. if it succeeds, we
+ // delete the non cfs files directly as they are not tracked anymore.
+ deleteNewFiles(filesToDelete);
+ merge.getMergeInfo().info.setUseCompoundFile(true);
+ }
+
+ // Have codec write SegmentInfo. Must do this after
+ // creating CFS so that 1) .si isn't slurped into CFS,
+ // and 2) .si reflects useCompoundFile=true change
+ // above:
+ codec.segmentInfoFormat().write(trackingDir, merge.getMergeInfo().info, context);
+ merge.getMergeInfo().info.addFiles(trackingDir.getCreatedFiles());
+ // Return without registering the segment files with IndexWriter.
+ // We do this together for all merges triggered by an addIndexes API,
+ // to keep the API transactional.
}
/** Copies the segment files as-is into the IndexWriter's directory. */
@@ -4819,19 +4992,21 @@ public class IndexWriter
suppressExceptions == false,
droppedSegment,
mr -> {
- final SegmentReader sr = mr.reader;
- final ReadersAndUpdates rld = getPooledInstance(sr.getOriginalSegmentInfo(), false);
- // We still hold a ref so it should not have been removed:
- assert rld != null;
- if (drop) {
- rld.dropChanges();
- } else {
- rld.dropMergingUpdates();
- }
- rld.release(sr);
- release(rld);
- if (drop) {
- readerPool.drop(rld.info);
+ if (merge.usesPooledReaders) {
+ final SegmentReader sr = mr.reader;
+ final ReadersAndUpdates rld = getPooledInstance(sr.getOriginalSegmentInfo(), false);
+ // We still hold a ref so it should not have been removed:
+ assert rld != null;
+ if (drop) {
+ rld.dropChanges();
+ } else {
+ rld.dropMergingUpdates();
+ }
+ rld.release(sr);
+ release(rld);
+ if (drop) {
+ readerPool.drop(rld.info);
+ }
}
});
} else {
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
index b84958ab609..a5d712bfdbb 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -193,6 +194,7 @@ public abstract class MergePolicy {
long mergeGen; // used by IndexWriter
boolean isExternal; // used by IndexWriter
int maxNumSegments = -1; // used by IndexWriter
+ boolean usesPooledReaders; // used by IndexWriter to drop readers while closing
/** Estimated size in bytes of the merged segment. */
public volatile long estimatedMergeBytes; // used by IndexWriter
@@ -229,6 +231,28 @@ public abstract class MergePolicy {
totalMaxDoc = segments.stream().mapToInt(i -> i.info.maxDoc()).sum();
mergeProgress = new OneMergeProgress();
mergeReaders = List.of();
+ usesPooledReaders = true;
+ }
+
+ /**
+ * Create a OneMerge directly from CodecReaders. Used to merge incoming readers in {@link
+ * IndexWriter#addIndexes(CodecReader...)}. This OneMerge works directly on readers and has an
+ * empty segments list.
+ *
+ * @param codecReaders Codec readers to merge
+ */
+ public OneMerge(CodecReader... codecReaders) {
+ List<MergeReader> readers = new ArrayList<>(codecReaders.length);
+ int totalDocs = 0;
+ for (CodecReader r : codecReaders) {
+ readers.add(new MergeReader(r, r.getLiveDocs()));
+ totalDocs += r.numDocs();
+ }
+ mergeReaders = List.copyOf(readers);
+ segments = List.of();
+ totalMaxDoc = totalDocs;
+ mergeProgress = new OneMergeProgress();
+ usesPooledReaders = false;
}
/**
@@ -472,15 +496,31 @@ public abstract class MergePolicy {
return b.toString();
}
+ CompletableFuture<Void> getMergeCompletedFutures() {
+ return CompletableFuture.allOf(
+ merges.stream()
+ .map(m -> m.mergeCompleted)
+ .collect(Collectors.toList())
+ .toArray(CompletableFuture<?>[]::new));
+ }
+
+ /** Waits, until interrupted, for all merges to complete. */
+ boolean await() {
+ try {
+ CompletableFuture<Void> future = getMergeCompletedFutures();
+ future.get();
+ return true;
+ } catch (InterruptedException e) {
+ throw new ThreadInterruptedException(e);
+ } catch (@SuppressWarnings("unused") ExecutionException | CancellationException e) {
+ return false;
+ }
+ }
+
/** Waits if necessary for at most the given time for all merges. */
boolean await(long timeout, TimeUnit unit) {
try {
- CompletableFuture<Void> future =
- CompletableFuture.allOf(
- merges.stream()
- .map(m -> m.mergeCompleted)
- .collect(Collectors.toList())
- .toArray(CompletableFuture<?>[]::new));
+ CompletableFuture<Void> future = getMergeCompletedFutures();
future.get(timeout, unit);
return true;
} catch (InterruptedException e) {
@@ -569,6 +609,24 @@ public abstract class MergePolicy {
MergeTrigger mergeTrigger, SegmentInfos segmentInfos, MergeContext mergeContext)
throws IOException;
+ /**
+ * Define the set of merge operations to perform on provided codec readers in {@link
+ * IndexWriter#addIndexes(CodecReader...)}.
+ *
+ * <p>The merge operation is required to convert provided readers into segments that can be added
+ * to the writer. This API can be overridden in custom merge policies to control the concurrency
+ * for addIndexes. Default implementation creates a single merge operation for all provided
+ * readers (lowest concurrency). Creating a merge for each reader, would provide the highest level
+ * of concurrency possible with the configured merge scheduler.
+ *
+ * @param readers CodecReader(s) to merge into the main index
+ */
+ public MergeSpecification findMerges(CodecReader... readers) throws IOException {
+ MergeSpecification mergeSpec = new MergeSpecification();
+ mergeSpec.add(new OneMerge(readers));
+ return mergeSpec;
+ }
+
/**
* Determine what set of merge operations is necessary in order to merge to {@code <=} the
* specified segment count. {@link IndexWriter} calls this when its {@link IndexWriter#forceMerge}
@@ -844,12 +902,24 @@ public abstract class MergePolicy {
}
static final class MergeReader {
+ final CodecReader codecReader;
final SegmentReader reader;
final Bits hardLiveDocs;
MergeReader(SegmentReader reader, Bits hardLiveDocs) {
+ this.codecReader = reader;
this.reader = reader;
this.hardLiveDocs = hardLiveDocs;
}
+
+ MergeReader(CodecReader reader, Bits hardLiveDocs) {
+ if (SegmentReader.class.isAssignableFrom(reader.getClass())) {
+ this.reader = (SegmentReader) reader;
+ } else {
+ this.reader = null;
+ }
+ this.codecReader = reader;
+ this.hardLiveDocs = hardLiveDocs;
+ }
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java b/lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java
index b6b06988ef1..ca23658592a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergeTrigger.java
@@ -43,4 +43,6 @@ public enum MergeTrigger {
COMMIT,
/** Merge was triggered on opening NRT readers. */
GET_READER,
+ /** Merge was triggered by an {@link IndexWriter#addIndexes(CodecReader...)} operation */
+ ADD_INDEXES,
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java
index 3429b6174d0..ce7b1ce006b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java
@@ -39,6 +39,16 @@ public final class NoMergePolicy extends MergePolicy {
return null;
}
+ @Override
+ public MergeSpecification findMerges(CodecReader... readers) throws IOException {
+ // addIndexes(CodecReader...) now uses MergePolicy and MergeScheduler to add
+ // provided readers (LUCENE-10216). We retain the default behavior here to enable
+ // addIndexes for consumers like IndexRearranger.
+ // This does not merge existing segments, but uses SegmentMerger to add
+ // new incoming readers to the index.
+ return super.findMerges(readers);
+ }
+
@Override
public MergeSpecification findForcedMerges(
SegmentInfos segmentInfos,
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
index 45d42b86d6c..c49aabfc166 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
@@ -20,7 +20,9 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.PostingsFormat;
@@ -683,6 +685,290 @@ public class TestAddIndexes extends LuceneTestCase {
writer.addDocument(doc);
}
+ private class ConcurrentAddIndexesMergePolicy extends TieredMergePolicy {
+
+ @Override
+ public MergeSpecification findMerges(CodecReader... readers) throws IOException {
+ // create a oneMerge for each reader to let them get concurrently processed by addIndexes()
+ MergeSpecification mergeSpec = new MergeSpecification();
+ for (CodecReader reader : readers) {
+ mergeSpec.add(new OneMerge(reader));
+ }
+ if (VERBOSE) {
+ System.out.println(
+ "No. of addIndexes merges returned by MergePolicy: " + mergeSpec.merges.size());
+ }
+ return mergeSpec;
+ }
+ }
+
+ private class AddIndexesWithReadersSetup {
+ Directory dir, destDir;
+ IndexWriter destWriter;
+ final DirectoryReader[] readers;
+ final int ADDED_DOCS_PER_READER = 15;
+ final int INIT_DOCS = 25;
+ final int NUM_READERS = 15;
+
+ public AddIndexesWithReadersSetup(MergeScheduler ms, MergePolicy mp) throws IOException {
+ dir = new MockDirectoryWrapper(random(), new ByteBuffersDirectory());
+ IndexWriter writer =
+ new IndexWriter(
+ dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(2));
+ for (int i = 0; i < ADDED_DOCS_PER_READER; i++) addDoc(writer);
+ writer.close();
+
+ destDir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setMergePolicy(mp);
+ iwc.setMergeScheduler(ms);
+ destWriter = new IndexWriter(destDir, iwc);
+ for (int i = 0; i < INIT_DOCS; i++) addDoc(destWriter);
+ destWriter.commit();
+
+ readers = new DirectoryReader[NUM_READERS];
+ for (int i = 0; i < NUM_READERS; i++) readers[i] = DirectoryReader.open(dir);
+ }
+
+ public void closeAll() throws IOException {
+ destWriter.close();
+ for (int i = 0; i < NUM_READERS; i++) readers[i].close();
+ destDir.close();
+ dir.close();
+ }
+ }
+
+ public void testAddIndexesWithConcurrentMerges() throws Exception {
+ ConcurrentAddIndexesMergePolicy mp = new ConcurrentAddIndexesMergePolicy();
+ AddIndexesWithReadersSetup c =
+ new AddIndexesWithReadersSetup(new ConcurrentMergeScheduler(), mp);
+ TestUtil.addIndexesSlowly(c.destWriter, c.readers);
+ c.destWriter.commit();
+ try (IndexReader reader = DirectoryReader.open(c.destDir)) {
+ assertEquals(c.INIT_DOCS + c.NUM_READERS * c.ADDED_DOCS_PER_READER, reader.numDocs());
+ }
+ c.closeAll();
+ }
+
+ private class PartialMergeScheduler extends MergeScheduler {
+
+ int mergesToDo;
+ int mergesTriggered = 0;
+
+ public PartialMergeScheduler(int mergesToDo) {
+ this.mergesToDo = mergesToDo;
+ if (VERBOSE) {
+ System.out.println(
+ "PartialMergeScheduler configured to mark all merges as failed, "
+ + "after triggering ["
+ + mergesToDo
+ + "] merges.");
+ }
+ }
+
+ @Override
+ public void merge(MergeSource mergeSource, MergeTrigger trigger) throws IOException {
+ while (true) {
+ MergePolicy.OneMerge merge = mergeSource.getNextMerge();
+ if (merge == null) {
+ break;
+ }
+ if (mergesTriggered >= mergesToDo) {
+ merge.close(false, false, mr -> {});
+ mergeSource.onMergeFinished(merge);
+ } else {
+ mergeSource.merge(merge);
+ mergesTriggered++;
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {}
+ }
+
+ public void testAddIndexesWithPartialMergeFailures() throws Exception {
+ final List<MergePolicy.OneMerge> merges = new ArrayList<>();
+ ConcurrentAddIndexesMergePolicy mp =
+ new ConcurrentAddIndexesMergePolicy() {
+ @Override
+ public MergeSpecification findMerges(CodecReader... readers) throws IOException {
+ MergeSpecification spec = super.findMerges(readers);
+ merges.addAll(spec.merges);
+ return spec;
+ }
+ };
+ AddIndexesWithReadersSetup c = new AddIndexesWithReadersSetup(new PartialMergeScheduler(2), mp);
+ assertThrows(RuntimeException.class, () -> TestUtil.addIndexesSlowly(c.destWriter, c.readers));
+ c.destWriter.commit();
+
+ // verify no docs got added and all interim files from successful merges have been deleted
+ try (IndexReader reader = DirectoryReader.open(c.destDir)) {
+ assertEquals(c.INIT_DOCS, reader.numDocs());
+ }
+ for (MergePolicy.OneMerge merge : merges) {
+ if (merge.getMergeInfo() != null) {
+ assertFalse(
+ Arrays.stream(c.destDir.listAll())
+ .collect(Collectors.toSet())
+ .containsAll(merge.getMergeInfo().files()));
+ }
+ }
+ c.closeAll();
+ }
+
+ public void testAddIndexesWithNullMergeSpec() throws Exception {
+ MergePolicy mp =
+ new TieredMergePolicy() {
+ @Override
+ public MergeSpecification findMerges(CodecReader... readers) throws IOException {
+ return null;
+ }
+ };
+ AddIndexesWithReadersSetup c =
+ new AddIndexesWithReadersSetup(new ConcurrentMergeScheduler(), mp);
+ TestUtil.addIndexesSlowly(c.destWriter, c.readers);
+ c.destWriter.commit();
+ try (IndexReader reader = DirectoryReader.open(c.destDir)) {
+ assertEquals(c.INIT_DOCS, reader.numDocs());
+ }
+ c.closeAll();
+ }
+
+ public void testAddIndexesWithEmptyMergeSpec() throws Exception {
+ MergePolicy mp =
+ new TieredMergePolicy() {
+ @Override
+ public MergeSpecification findMerges(CodecReader... readers) throws IOException {
+ return new MergeSpecification();
+ }
+ };
+ AddIndexesWithReadersSetup c =
+ new AddIndexesWithReadersSetup(new ConcurrentMergeScheduler(), mp);
+ TestUtil.addIndexesSlowly(c.destWriter, c.readers);
+ c.destWriter.commit();
+ try (IndexReader reader = DirectoryReader.open(c.destDir)) {
+ assertEquals(c.INIT_DOCS, reader.numDocs());
+ }
+ c.closeAll();
+ }
+
+ private class CountingSerialMergeScheduler extends MergeScheduler {
+
+ int explicitMerges = 0;
+ int addIndexesMerges = 0;
+
+ @Override
+ public void merge(MergeSource mergeSource, MergeTrigger trigger) throws IOException {
+ while (true) {
+ MergePolicy.OneMerge merge = mergeSource.getNextMerge();
+ if (merge == null) {
+ break;
+ }
+ mergeSource.merge(merge);
+ if (trigger == MergeTrigger.EXPLICIT) {
+ explicitMerges++;
+ }
+ if (trigger == MergeTrigger.ADD_INDEXES) {
+ addIndexesMerges++;
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {}
+ }
+
+ public void testAddIndexesWithEmptyReaders() throws Exception {
+ Directory destDir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setMergePolicy(new ConcurrentAddIndexesMergePolicy());
+ CountingSerialMergeScheduler ms = new CountingSerialMergeScheduler();
+ iwc.setMergeScheduler(ms);
+ IndexWriter destWriter = new IndexWriter(destDir, iwc);
+ final int initialDocs = 15;
+ for (int i = 0; i < initialDocs; i++) addDoc(destWriter);
+ destWriter.commit();
+
+ // create empty readers
+ Directory dir = new MockDirectoryWrapper(random(), new ByteBuffersDirectory());
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
+ writer.close();
+ final int numReaders = 20;
+ DirectoryReader[] readers = new DirectoryReader[numReaders];
+ for (int i = 0; i < numReaders; i++) readers[i] = DirectoryReader.open(dir);
+
+ TestUtil.addIndexesSlowly(destWriter, readers);
+ destWriter.commit();
+
+ // verify no docs were added
+ try (IndexReader reader = DirectoryReader.open(destDir)) {
+ assertEquals(initialDocs, reader.numDocs());
+ }
+ // verify no merges were triggered
+ assertEquals(0, ms.addIndexesMerges);
+
+ destWriter.close();
+ for (int i = 0; i < numReaders; i++) readers[i].close();
+ destDir.close();
+ dir.close();
+ }
+
+ public void testCascadingMergesTriggered() throws Exception {
+ ConcurrentAddIndexesMergePolicy mp = new ConcurrentAddIndexesMergePolicy();
+ CountingSerialMergeScheduler ms = new CountingSerialMergeScheduler();
+ AddIndexesWithReadersSetup c = new AddIndexesWithReadersSetup(ms, mp);
+ TestUtil.addIndexesSlowly(c.destWriter, c.readers);
+ assertTrue(ms.explicitMerges > 0);
+ c.closeAll();
+ }
+
+ public void testAddIndexesHittingMaxDocsLimit() throws Exception {
+ final int writerMaxDocs = 15;
+ IndexWriter.setMaxDocs(writerMaxDocs);
+
+ // create destination writer
+ Directory destDir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setMergePolicy(new ConcurrentAddIndexesMergePolicy());
+ CountingSerialMergeScheduler ms = new CountingSerialMergeScheduler();
+ iwc.setMergeScheduler(ms);
+ IndexWriter destWriter = new IndexWriter(destDir, iwc);
+ for (int i = 0; i < writerMaxDocs; i++) addDoc(destWriter);
+ destWriter.commit();
+
+ // create readers to add
+ Directory dir = new MockDirectoryWrapper(random(), new ByteBuffersDirectory());
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
+ for (int i = 0; i < 10; i++) addDoc(writer);
+ writer.close();
+ final int numReaders = 20;
+ DirectoryReader[] readers = new DirectoryReader[numReaders];
+ for (int i = 0; i < numReaders; i++) readers[i] = DirectoryReader.open(dir);
+
+ boolean success = false;
+ try {
+ TestUtil.addIndexesSlowly(destWriter, readers);
+ success = true;
+ } catch (IllegalArgumentException e) {
+ assertTrue(
+ e.getMessage()
+ .contains("number of documents in the index cannot exceed " + writerMaxDocs));
+ }
+ assertFalse(success);
+
+ // verify no docs were added
+ destWriter.commit();
+ try (IndexReader reader = DirectoryReader.open(destDir)) {
+ assertEquals(writerMaxDocs, reader.numDocs());
+ }
+
+ destWriter.close();
+ for (int i = 0; i < numReaders; i++) readers[i].close();
+ destDir.close();
+ dir.close();
+ }
+
private abstract class RunAddIndexesThreads {
Directory dir, dir2;
@@ -704,7 +990,10 @@ public class TestAddIndexes extends LuceneTestCase {
writer.close();
dir2 = newDirectory();
- writer2 = new IndexWriter(dir2, new IndexWriterConfig(new MockAnalyzer(random())));
+ MergePolicy mp = new ConcurrentAddIndexesMergePolicy();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setMergePolicy(mp);
+ writer2 = new IndexWriter(dir2, iwc);
writer2.commit();
readers = new DirectoryReader[NUM_COPY];
@@ -985,7 +1274,6 @@ public class TestAddIndexes extends LuceneTestCase {
System.out.println("TEST: done join threads");
}
c.closeDir();
-
assertTrue(c.failures.size() == 0);
}
@@ -1014,7 +1302,6 @@ public class TestAddIndexes extends LuceneTestCase {
}
c.closeDir();
-
assertTrue(c.failures.size() == 0);
}
@@ -1488,7 +1775,9 @@ public class TestAddIndexes extends LuceneTestCase {
}
writer.commit();
writer.close();
+
DirectoryReader reader = DirectoryReader.open(dir1);
+ // wrappedReader filters out soft deleted docs
DirectoryReader wrappedReader = new SoftDeletesDirectoryReaderWrapper(reader, "soft_delete");
Directory dir2 = newDirectory();
int numDocs = reader.numDocs();
@@ -1504,9 +1793,13 @@ public class TestAddIndexes extends LuceneTestCase {
assertEquals(wrappedReader.numDocs(), writer.getDocStats().numDocs);
assertEquals(maxDoc, writer.getDocStats().maxDoc);
writer.commit();
- SegmentCommitInfo commitInfo = writer.cloneSegmentInfos().info(0);
- assertEquals(maxDoc - wrappedReader.numDocs(), commitInfo.getSoftDelCount());
+ int softDeleteCount =
+ writer.cloneSegmentInfos().asList().stream()
+ .mapToInt(SegmentCommitInfo::getSoftDelCount)
+ .sum();
+ assertEquals(maxDoc - wrappedReader.numDocs(), softDeleteCount);
writer.close();
+
Directory dir3 = newDirectory();
iwc1 = newIndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("soft_delete");
writer = new IndexWriter(dir3, iwc1);
@@ -1517,6 +1810,7 @@ public class TestAddIndexes extends LuceneTestCase {
}
writer.addIndexes(readers);
assertEquals(wrappedReader.numDocs(), writer.getDocStats().numDocs);
+ // soft deletes got filtered out when wrappedReader(s) were added.
assertEquals(wrappedReader.numDocs(), writer.getDocStats().maxDoc);
IOUtils.close(reader, writer, dir3, dir2, dir1);
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
index 0847ff63fd1..19d3284b11d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
@@ -352,7 +352,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase {
done = true;
}
- } catch (IllegalStateException | IOException e) {
+ } catch (IllegalStateException | IOException | MergePolicy.MergeException e) {
success = false;
err = e;
if (VERBOSE) {
@@ -360,7 +360,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase {
e.printStackTrace(System.out);
}
- if (1 == x) {
+ if (1 == x && (e instanceof MergePolicy.MergeException == false)) {
e.printStackTrace(System.out);
fail(methodName + " hit IOException after disk space was freed up");
}