You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/01/09 17:44:44 UTC
svn commit: r1650594 - in /lucene/dev/branches/branch_5x/lucene/core/src:
java/org/apache/lucene/index/ test/org/apache/lucene/index/
Author: mikemccand
Date: Fri Jan 9 16:44:44 2015
New Revision: 1650594
URL: http://svn.apache.org/r1650594
Log:
LUCENE-6119: must check merge for abort even when we are not rate limiting; don't wrap rate limiter when doing addIndexes (it's not abortable); don't leak file handle when wrapping
Modified:
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/MergeRateLimiter.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1650594&r1=1650593&r2=1650594&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Fri Jan 9 16:44:44 2015
@@ -394,8 +394,10 @@ public class ConcurrentMergeScheduler ex
/**
* Returns the number of merge threads that are alive. Note that this number
* is ≤ {@link #mergeThreads} size.
+ *
+ * @lucene.internal
*/
- protected synchronized int mergeThreadCount() {
+ public synchronized int mergeThreadCount() {
int count = 0;
for (MergeThread mergeThread : mergeThreads) {
if (mergeThread.isAlive() && mergeThread.merge.rateLimiter.getAbort() == false) {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1650594&r1=1650593&r2=1650594&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Jan 9 16:44:44 2015
@@ -2537,7 +2537,7 @@ public class IndexWriter implements Clos
// TODO: somehow we should fix this merge so it's
// abortable so that IW.close(false) is able to stop it
- TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(mergeDirectory);
+ TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
SegmentInfo info = new SegmentInfo(directory, Version.LATEST, mergedName, -1,
false, codec, null, StringHelper.randomId(), new HashMap<String,String>());
@@ -4684,14 +4684,18 @@ public class IndexWriter implements Clos
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
+ // Paranoia defense: if this trips we have a bug somewhere...
+ IndexWriter.this.ensureOpen(false);
+
// This Directory is only supposed to be used during merging,
// so all writes should have MERGE context, else there is a bug
// somewhere that is failing to pass down the right IOContext:
assert context.context == IOContext.Context.MERGE: "got context=" + context.context;
- IndexOutput output = in.createOutput(name, context);
+
MergeRateLimiter rateLimiter = rateLimiters.get();
assert rateLimiter != null;
- return new RateLimitedIndexOutput(rateLimiter, output);
+
+ return new RateLimitedIndexOutput(rateLimiter, in.createOutput(name, context));
}
};
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/MergeRateLimiter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/MergeRateLimiter.java?rev=1650594&r1=1650593&r2=1650594&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/MergeRateLimiter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/MergeRateLimiter.java Fri Jan 9 16:44:44 2015
@@ -125,6 +125,10 @@ public class MergeRateLimiter extends Ra
/** Returns NO if no pause happened, STOPPED if pause because rate was 0.0 (merge is stopped), PAUSED if paused with a normal rate limit. */
private synchronized PauseResult maybePause(long bytes, long curNS) throws MergePolicy.MergeAbortedException {
+
+ // Now is a good time to abort the merge:
+ checkAbort();
+
double secondsToPause = (bytes/1024./1024.) / mbPerSec;
// Time we should sleep until; this is purely instantaneous
@@ -150,9 +154,6 @@ public class MergeRateLimiter extends Ra
int sleepMS = (int) (curPauseNS / 1000000);
int sleepNS = (int) (curPauseNS % 1000000);
- // Now is a good time to abort the merge:
- checkAbort();
-
double rate = mbPerSec;
try {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1650594&r1=1650593&r2=1650594&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Jan 9 16:44:44 2015
@@ -955,10 +955,16 @@ public class TestAddIndexes extends Luce
System.out.println("TEST: now force rollback");
}
c.didClose = true;
+ MergeScheduler ms = c.writer2.getConfig().getMergeScheduler();
+
c.writer2.rollback();
c.joinThreads();
+ if (ms instanceof ConcurrentMergeScheduler) {
+ assertEquals(0, ((ConcurrentMergeScheduler) ms).mergeThreadCount());
+ }
+
c.closeDir();
assertTrue(c.failures.size() == 0);