You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2020/11/24 12:13:02 UTC
[lucene-solr] branch branch_8x updated: LUCENE-9508: Fix
DocumentsWriter to block threads until unstalled (#2085)
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 9057bb9 LUCENE-9508: Fix DocumentsWriter to block threads until unstalled (#2085)
9057bb9 is described below
commit 9057bb927bef9b11ab70c612b3efa60fd4a0477d
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Tue Nov 24 13:05:21 2020 +0100
LUCENE-9508: Fix DocumentsWriter to block threads until unstalled (#2085)
DWStallControl expects the caller to loop on top of the wait call to make
progress with flushing if the DW is stalled. This logic wasn't applied such that
DW only stalled for one second and then released the indexing thread. This can cause
OOM if for instance during a full flush one DWPT gets stuck and onther threads keep on
indexing.
---
lucene/CHANGES.txt | 5 +++
.../org/apache/lucene/index/DocumentsWriter.java | 20 ++++------
.../lucene/index/DocumentsWriterStallControl.java | 4 ++
.../org/apache/lucene/index/TestIndexWriter.java | 43 ++++++++++++++++++++++
4 files changed, 60 insertions(+), 12 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5aeccb3..cebafc7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -42,6 +42,11 @@ Optimizations
Bug Fixes
---------------------
+* LUCENE-9508: DocumentsWriter was only stalling threads for 1 second allowing
+ documents to be indexed even the DocumentsWriter wasn't able to keep up flushing.
+ Unless IW can't make progress due to an ill behaving DWPT this issue was barely
+ noticeable. (Simon Willnauer)
+
* LUCENE-9581: Japanese tokenizer should discard the compound token instead of disabling the decomposition
of long tokens when discardCompoundToken is activated. (Jim Ferenczi)
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
index 9f01f88..3da9a76 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
@@ -371,19 +371,15 @@ final class DocumentsWriter implements Closeable, Accountable {
private boolean preUpdate() throws IOException {
ensureOpen();
boolean hasEvents = false;
-
- if (flushControl.anyStalledThreads() || (flushControl.numQueuedFlushes() > 0 && config.checkPendingFlushOnUpdate)) {
+ while (flushControl.anyStalledThreads() || (flushControl.numQueuedFlushes() > 0 && config.checkPendingFlushOnUpdate)) {
// Help out flushing any queued DWPTs so we can un-stall:
- do {
- // Try pick up pending threads here if possible
- DocumentsWriterPerThread flushingDWPT;
- while ((flushingDWPT = flushControl.nextPendingFlush()) != null) {
- // Don't push the delete here since the update could fail!
- hasEvents |= doFlush(flushingDWPT);
- }
-
- flushControl.waitIfStalled(); // block if stalled
- } while (flushControl.numQueuedFlushes() != 0); // still queued DWPTs try help flushing
+ // Try pick up pending threads here if possible
+ DocumentsWriterPerThread flushingDWPT;
+ while ((flushingDWPT = flushControl.nextPendingFlush()) != null) {
+ // Don't push the delete here since the update could fail!
+ hasEvents |= doFlush(flushingDWPT);
+ }
+ flushControl.waitIfStalled(); // block if stalled
}
return hasEvents;
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java
index 30a9277..99d2748 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterStallControl.java
@@ -103,6 +103,10 @@ final class DocumentsWriterStallControl {
synchronized boolean hasBlocked() { // for tests
return numWaiting > 0;
}
+
+ synchronized int getNumWaiting() { // for tests
+ return numWaiting;
+ }
boolean isHealthy() { // for tests
return !stalled; // volatile read!
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 1761e1c..2eb5e41 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -4259,4 +4259,47 @@ public class TestIndexWriter extends LuceneTestCase {
}
}
}
+
+ public void testIndexWriterBlocksOnStall() throws IOException, InterruptedException {
+ try (Directory dir = newDirectory()) {
+ try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig())) {
+ DocumentsWriterStallControl stallControl = writer.getDocsWriter().flushControl.stallControl;
+ stallControl.updateStalled(true);
+ Thread[] threads = new Thread[random().nextInt(3)+1];
+ AtomicLong numThreadsCompleted = new AtomicLong(0);
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread(() -> {
+ Document d = new Document();
+ d.add(new StringField("id", Integer.toString(0), Field.Store.YES));
+ try {
+ writer.addDocument(d);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ numThreadsCompleted.incrementAndGet();
+ });
+ threads[i].start();
+ }
+ try {
+ for (int i = 0; i < 10; i++) {
+ synchronized (stallControl) {
+ stallControl.notifyAll();
+ }
+ while (stallControl.getNumWaiting() != threads.length) {
+ // wait for all threads to be stalled again
+ assertEquals(0, writer.getPendingNumDocs());
+ assertEquals(0, numThreadsCompleted.get());
+ }
+ }
+ } finally {
+ stallControl.updateStalled(false);
+ for (Thread t : threads) {
+ t.join();
+ }
+ }
+ writer.commit();
+ assertEquals(threads.length, writer.getDocStats().maxDoc);
+ }
+ }
+ }
}