You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rj...@apache.org on 2014/08/18 19:16:05 UTC
svn commit: r1618667 - in /lucene/dev/trunk/lucene/core/src:
java/org/apache/lucene/index/IndexWriter.java
test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
Author: rjernst
Date: Mon Aug 18 17:16:04 2014
New Revision: 1618667
URL: http://svn.apache.org/r1618667
Log:
Fix waitForMerges to allow merge schedule to run one last time
Modified:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1618667&r1=1618666&r2=1618667&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Mon Aug 18 17:16:04 2014
@@ -908,7 +908,7 @@ public class IndexWriter implements Clos
infoStream.message("IW", "now flush at close");
}
flush(true, true);
- finishMerges(true);
+ waitForMerges();
commitInternal(config.getMergePolicy());
rollbackInternal(); // ie close, since we just committed
success = true;
@@ -1999,7 +1999,7 @@ public class IndexWriter implements Clos
try {
synchronized(this) {
- finishMerges(false);
+ abortMerges();
stopMerges = true;
}
@@ -2136,7 +2136,7 @@ public class IndexWriter implements Clos
synchronized (this) {
try {
// Abort any running merges
- finishMerges(false);
+ abortMerges();
// Remove all segments
segmentInfos.clear();
// Ask deleter to locate unreferenced files & remove them:
@@ -2173,60 +2173,45 @@ public class IndexWriter implements Clos
/** Aborts running merges. Be careful when using this
* method: when you abort a long-running merge, you lose
* a lot of work that must later be redone. */
- public void abortMerges() {
- finishMerges(false);
- }
-
- private synchronized void finishMerges(boolean waitForMerges) {
- if (!waitForMerges) {
+ public synchronized void abortMerges() {
+ stopMerges = true;
- stopMerges = true;
-
- // Abort all pending & running merges:
- for (final MergePolicy.OneMerge merge : pendingMerges) {
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "now abort pending merge " + segString(merge.segments));
- }
- merge.abort();
- mergeFinish(merge);
+ // Abort all pending & running merges:
+ for (final MergePolicy.OneMerge merge : pendingMerges) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "now abort pending merge " + segString(merge.segments));
}
- pendingMerges.clear();
+ merge.abort();
+ mergeFinish(merge);
+ }
+ pendingMerges.clear();
- for (final MergePolicy.OneMerge merge : runningMerges) {
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "now abort running merge " + segString(merge.segments));
- }
- merge.abort();
+ for (final MergePolicy.OneMerge merge : runningMerges) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "now abort running merge " + segString(merge.segments));
}
+ merge.abort();
+ }
- // These merges periodically check whether they have
- // been aborted, and stop if so. We wait here to make
- // sure they all stop. It should not take very long
- // because the merge threads periodically check if
- // they are aborted.
- while(runningMerges.size() > 0) {
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "now wait for " + runningMerges.size() + " running merge/s to abort");
- }
- doWait();
+ // These merges periodically check whether they have
+ // been aborted, and stop if so. We wait here to make
+ // sure they all stop. It should not take very long
+ // because the merge threads periodically check if
+ // they are aborted.
+ while(runningMerges.size() > 0) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "now wait for " + runningMerges.size() + " running merge/s to abort");
}
+ doWait();
+ }
- stopMerges = false;
- notifyAll();
-
- assert 0 == mergingSegments.size();
+ stopMerges = false;
+ notifyAll();
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "all running merges have aborted");
- }
+ assert 0 == mergingSegments.size();
- } else {
- // waitForMerges() will ensure any running addIndexes finishes.
- // It's fine if a new one attempts to start because from our
- // caller above the call will see that we are in the
- // process of closing, and will throw an
- // AlreadyClosedException.
- waitForMerges();
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "all running merges have aborted");
}
}
@@ -2236,20 +2221,30 @@ public class IndexWriter implements Clos
* <p>It is guaranteed that any merges started prior to calling this method
* will have completed once this method completes.</p>
*/
- public synchronized void waitForMerges() {
- ensureOpen(false);
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "waitForMerges");
- }
- while(pendingMerges.size() > 0 || runningMerges.size() > 0) {
- doWait();
- }
+ public void waitForMerges() throws IOException {
- // sanity check
- assert 0 == mergingSegments.size();
+ // Give merge scheduler last chance to run, in case
+ // any pending merges are waiting. We can't hold IW's lock
+ // when going into merge because it can lead to deadlock.
+ mergeScheduler.merge(this, MergeTrigger.CLOSING, false);
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "waitForMerges done");
+ synchronized (this) {
+ ensureOpen(false);
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "waitForMerges");
+ }
+
+
+ while (pendingMerges.size() > 0 || runningMerges.size() > 0) {
+ doWait();
+ }
+
+ // sanity check
+ assert 0 == mergingSegments.size();
+
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "waitForMerges done");
+ }
}
}
@@ -3835,7 +3830,7 @@ public class IndexWriter implements Clos
* the synchronized lock on IndexWriter instance. */
final synchronized void mergeFinish(MergePolicy.OneMerge merge) {
- // forceMerge, addIndexes or finishMerges may be waiting
+ // forceMerge, addIndexes or waitForMerges may be waiting
// on merges to finish.
notifyAll();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java?rev=1618667&r1=1618666&r2=1618667&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java Mon Aug 18 17:16:04 2014
@@ -233,7 +233,7 @@ public class TestIndexWriterMergePolicy
writer.addDocument(doc);
}
- private void checkInvariants(IndexWriter writer) {
+ private void checkInvariants(IndexWriter writer) throws IOException {
writer.waitForMerges();
int maxBufferedDocs = writer.getConfig().getMaxBufferedDocs();
int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor();
Re: svn commit: r1618667 - in /lucene/dev/trunk/lucene/core/src:
java/org/apache/lucene/index/IndexWriter.java
test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
Posted by Chris Hostetter <ho...@fucit.org>.
: This was not an issue in 4.9. It was introduced during refactorings for 4.10.
cool - thanks.
:
: On Mon, Aug 18, 2014 at 10:24 AM, Chris Hostetter
: <ho...@fucit.org> wrote:
: > : Log:
: > : Fix waitForMerges to allow merge schedule to run one last time
: >
: > how old is the code being fixed? If it was in 4.9 this fix should
: > probably have a tracking jira & be listed in CHANGES.txt.
: >
: >
: > -Hoss
: > http://www.lucidworks.com/
: >
: > ---------------------------------------------------------------------
: > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
: > For additional commands, e-mail: dev-help@lucene.apache.org
: >
:
: ---------------------------------------------------------------------
: To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
: For additional commands, e-mail: dev-help@lucene.apache.org
:
:
-Hoss
http://www.lucidworks.com/
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1618667 - in /lucene/dev/trunk/lucene/core/src:
java/org/apache/lucene/index/IndexWriter.java test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
Posted by Ryan Ernst <ry...@iernst.net>.
This was not an issue in 4.9. It was introduced during refactorings for 4.10.
On Mon, Aug 18, 2014 at 10:24 AM, Chris Hostetter
<ho...@fucit.org> wrote:
> : Log:
> : Fix waitForMerges to allow merge schedule to run one last time
>
> how old is the code being fixed? If it was in 4.9 this fix should
> probably have a tracking jira & be listed in CHANGES.txt.
>
>
> -Hoss
> http://www.lucidworks.com/
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1618667 - in /lucene/dev/trunk/lucene/core/src:
java/org/apache/lucene/index/IndexWriter.java
test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
Posted by Chris Hostetter <ho...@fucit.org>.
: Log:
: Fix waitForMerges to allow merge schedule to run one last time
how old is the code being fixed? If it was in 4.9 this fix should
probably have a tracking jira & be listed in CHANGES.txt.
-Hoss
http://www.lucidworks.com/
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org