You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/10/08 22:38:58 UTC
svn commit: r1530416 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/java/org/apache/lucene/util/
lucene/core/src/test/org/apache/lucene/index/ lucene/facet/
lucene/facet/src/te...
Author: mikemccand
Date: Tue Oct 8 20:38:57 2013
New Revision: 1530416
URL: http://svn.apache.org/r1530416
Log:
LUCENE-5263: fix cases where transient IOExc (e.g. due to disk full, file descriptor exhaustion) in IndexWriter could lead to silently losing deletions
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java
lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java
lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Tue Oct 8 20:38:57 2013
@@ -63,6 +63,10 @@ Bug Fixes
* LUCENE-5262: Fixed file handle leaks when multiple attempts to open an
NRT reader hit exceptions. (Shai Erera)
+* LUCENE-5263: Transient IOExceptions, e.g. due to disk full or file
+ descriptor exhaustion, hit at unlucky times inside IndexWriter could
+ lead to silently losing deletions. (Shai Erera, Mike McCandless)
+
API Changes:
* LUCENE-5222: Add SortField.needsScores(). Previously it was not possible
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Oct 8 20:38:57 2013
@@ -450,6 +450,10 @@ public class IndexWriter implements Clos
}
public synchronized void release(ReadersAndLiveDocs rld) throws IOException {
+ release(rld, true);
+ }
+
+ public synchronized void release(ReadersAndLiveDocs rld, boolean assertInfoLive) throws IOException {
// Matches incRef in get:
rld.decRef();
@@ -462,11 +466,17 @@ public class IndexWriter implements Clos
// pooling, so remove it:
if (rld.writeLiveDocs(directory)) {
// Make sure we only write del docs for a live segment:
- assert infoIsLive(rld.info);
- // Must checkpoint w/ deleter, because we just
- // created created new _X_N.del file.
- deleter.checkpoint(segmentInfos, false);
+ assert assertInfoLive == false || infoIsLive(rld.info);
+ // Must checkpoint because we just
+ // created new _X_N.del and field updates files;
+ // don't call IW.checkpoint because that also
+ // increments SIS.version, which we do not want to
+ // do here: it was done previously (after we
+ // invoked BDS.applyDeletes), whereas here all we
+ // did was move the state to disk:
+ checkpointNoSIS();
}
+ //System.out.println("IW: done writeLiveDocs for info=" + rld.info);
rld.dropReaders();
readerMap.remove(rld.info);
@@ -485,12 +495,19 @@ public class IndexWriter implements Clos
if (doSave && rld.writeLiveDocs(directory)) {
// Make sure we only write del docs for a live segment:
assert infoIsLive(rld.info);
- // Must checkpoint w/ deleter, because we just
- // created created new _X_N.del file.
- deleter.checkpoint(segmentInfos, false);
+ // Must checkpoint because we just
+ // created new _X_N.del and field updates files;
+ // don't call IW.checkpoint because that also
+ // increments SIS.version, which we do not want to
+ // do here: it was done previously (after we
+ // invoked BDS.applyDeletes), whereas here all we
+ // did was move the state to disk:
+ checkpointNoSIS();
}
} catch (Throwable t) {
- if (priorE != null) {
+ if (doSave) {
+ IOUtils.reThrow(t);
+ } else if (priorE == null) {
priorE = t;
}
}
@@ -508,15 +525,15 @@ public class IndexWriter implements Clos
try {
rld.dropReaders();
} catch (Throwable t) {
- if (priorE != null) {
+ if (doSave) {
+ IOUtils.reThrow(t);
+ } else if (priorE == null) {
priorE = t;
}
}
}
assert readerMap.size() == 0;
- if (priorE != null) {
- throw new RuntimeException(priorE);
- }
+ IOUtils.reThrow(priorE);
}
/**
@@ -533,9 +550,14 @@ public class IndexWriter implements Clos
if (rld.writeLiveDocs(directory)) {
// Make sure we only write del docs for a live segment:
assert infoIsLive(info);
- // Must checkpoint w/ deleter, because we just
- // created created new _X_N.del file.
- deleter.checkpoint(segmentInfos, false);
+ // Must checkpoint because we just
+ // created new _X_N.del and field updates files;
+ // don't call IW.checkpoint because that also
+ // increments SIS.version, which we do not want to
+ // do here: it was done previously (after we
+ // invoked BDS.applyDeletes), whereas here all we
+ // did was move the state to disk:
+ checkpointNoSIS();
}
}
}
@@ -1017,19 +1039,30 @@ public class IndexWriter implements Clos
infoStream.message("IW", "now call final commit()");
}
+ // Must do this before commitInternal, in case any of
+ // the dropped readers in the pool wrote a new live
+ // docs:
+ synchronized(this) {
+ readerPool.dropAll(true);
+ }
+
if (doFlush) {
commitInternal();
}
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "at close: " + segString());
+ synchronized(this) {
+ deleter.close();
}
+
// used by assert below
final DocumentsWriter oldWriter = docWriter;
- synchronized(this) {
- readerPool.dropAll(true);
+
+ synchronized (this) {
docWriter = null;
- deleter.close();
+ }
+
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "at close: " + segString());
}
if (writeLock != null) {
@@ -2236,6 +2269,15 @@ public class IndexWriter implements Clos
deleter.checkpoint(segmentInfos, false);
}
+ /** Checkpoints with IndexFileDeleter, so it's aware of
+ * new files, and increments changeCount, so on
+ * close/commit we will write a new segments file, but
+ * does NOT bump segmentInfos.version. */
+ synchronized void checkpointNoSIS() throws IOException {
+ changeCount++;
+ deleter.checkpoint(segmentInfos, false);
+ }
+
/** Called internally if any index state has changed. */
synchronized void changed() {
changeCount++;
@@ -2943,6 +2985,8 @@ public class IndexWriter implements Clos
segmentInfos.updateGeneration(pendingCommit);
lastCommitChangeCount = pendingCommitChangeCount;
rollbackSegments = pendingCommit.createBackupSegmentInfos();
+ // NOTE: don't use this.checkpoint() here, because
+ // we do not want to increment changeCount:
deleter.checkpoint(pendingCommit, true);
} finally {
// Matches the incRef done in prepareCommit:
@@ -3289,15 +3333,22 @@ public class IndexWriter implements Clos
assert merge.info.info.getDocCount() != 0 || keepFullyDeletedSegments || dropSegment;
- segmentInfos.applyMergeChanges(merge, dropSegment);
-
if (mergedDeletes != null) {
if (dropSegment) {
mergedDeletes.dropChanges();
}
- readerPool.release(mergedDeletes);
+ // Pass false for assertInfoLive because the merged
+ // segment is not yet live (only below do we commit it
+ // to the segmentInfos):
+ readerPool.release(mergedDeletes, false);
}
+ // Must do this after readerPool.release, in case an
+ // exception is hit e.g. writing the live docs for the
+ // merge segment, in which case we need to abort the
+ // merge:
+ segmentInfos.applyMergeChanges(merge, dropSegment);
+
if (dropSegment) {
assert !segmentInfos.contains(merge.info);
readerPool.drop(merge.info);
@@ -3361,17 +3412,12 @@ public class IndexWriter implements Clos
// in which case we must throw it so, for example, the
// rollbackTransaction code in addIndexes* is
// executed.
- if (merge.isExternal)
+ if (merge.isExternal) {
throw (MergePolicy.MergeAbortedException) t;
- } else if (t instanceof IOException)
- throw (IOException) t;
- else if (t instanceof RuntimeException)
- throw (RuntimeException) t;
- else if (t instanceof Error)
- throw (Error) t;
- else
- // Should not get here
- throw new RuntimeException(t);
+ }
+ } else {
+ IOUtils.reThrow(t);
+ }
}
/**
@@ -3686,11 +3732,8 @@ public class IndexWriter implements Clos
}
// If any error occured, throw it.
- if (!suppressExceptions && th != null) {
- if (th instanceof IOException) throw (IOException) th;
- if (th instanceof RuntimeException) throw (RuntimeException) th;
- if (th instanceof Error) throw (Error) th;
- throw new RuntimeException(th);
+ if (!suppressExceptions) {
+ IOUtils.reThrow(th);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java Tue Oct 8 20:38:57 2013
@@ -167,12 +167,7 @@ public class SegmentInfoPerCommit {
/** Returns a description of this segment. */
public String toString(Directory dir, int pendingDelCount) {
- return info.toString(dir, delCount + pendingDelCount);
- }
-
- @Override
- public String toString() {
- String s = info.toString(info.dir, delCount);
+ String s = info.toString(dir, delCount + pendingDelCount);
if (delGen != -1) {
s += ":delGen=" + delGen;
}
@@ -180,6 +175,11 @@ public class SegmentInfoPerCommit {
}
@Override
+ public String toString() {
+ return toString(info.dir, 0);
+ }
+
+ @Override
public SegmentInfoPerCommit clone() {
SegmentInfoPerCommit other = new SegmentInfoPerCommit(info, delCount, delGen);
// Not clear that we need to carry over nextWriteDelGen
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Tue Oct 8 20:38:57 2013
@@ -64,8 +64,9 @@ final class StandardDirectoryReader exte
} catch(IOException ex) {
prior = ex;
} finally {
- if (!success)
+ if (!success) {
IOUtils.closeWhileHandlingException(prior, readers);
+ }
}
}
return new StandardDirectoryReader(directory, readers, null, sis, termInfosIndexDivisor, false);
@@ -102,7 +103,7 @@ final class StandardDirectoryReader exte
readers.add(reader);
infosUpto++;
} else {
- reader.close();
+ reader.decRef();
segmentInfos.remove(infosUpto);
}
} finally {
@@ -210,12 +211,7 @@ final class StandardDirectoryReader exte
}
}
// throw the first exception
- if (prior != null) {
- if (prior instanceof IOException) throw (IOException) prior;
- if (prior instanceof RuntimeException) throw (RuntimeException) prior;
- if (prior instanceof Error) throw (Error) prior;
- throw new RuntimeException(prior);
- }
+ IOUtils.reThrow(prior);
}
}
return new StandardDirectoryReader(directory, newReaders, null, infos, termInfosIndexDivisor, false);
@@ -355,7 +351,9 @@ final class StandardDirectoryReader exte
try {
r.decRef();
} catch (Throwable t) {
- if (firstExc == null) firstExc = t;
+ if (firstExc == null) {
+ firstExc = t;
+ }
}
}
@@ -366,12 +364,7 @@ final class StandardDirectoryReader exte
}
// throw the first exception
- if (firstExc != null) {
- if (firstExc instanceof IOException) throw (IOException) firstExc;
- if (firstExc instanceof RuntimeException) throw (RuntimeException) firstExc;
- if (firstExc instanceof Error) throw (Error) firstExc;
- throw new RuntimeException(firstExc);
- }
+ IOUtils.reThrow(firstExc);
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Tue Oct 8 20:38:57 2013
@@ -90,11 +90,8 @@ public final class IOUtils {
if (priorException != null) {
throw priorException;
- } else if (th != null) {
- if (th instanceof IOException) throw (IOException) th;
- if (th instanceof RuntimeException) throw (RuntimeException) th;
- if (th instanceof Error) throw (Error) th;
- throw new RuntimeException(th);
+ } else {
+ reThrow(th);
}
}
@@ -119,11 +116,8 @@ public final class IOUtils {
if (priorException != null) {
throw priorException;
- } else if (th != null) {
- if (th instanceof IOException) throw (IOException) th;
- if (th instanceof RuntimeException) throw (RuntimeException) th;
- if (th instanceof Error) throw (Error) th;
- throw new RuntimeException(th);
+ } else {
+ reThrow(th);
}
}
@@ -153,12 +147,7 @@ public final class IOUtils {
}
}
- if (th != null) {
- if (th instanceof IOException) throw (IOException) th;
- if (th instanceof RuntimeException) throw (RuntimeException) th;
- if (th instanceof Error) throw (Error) th;
- throw new RuntimeException(th);
- }
+ reThrow(th);
}
/**
@@ -181,12 +170,7 @@ public final class IOUtils {
}
}
- if (th != null) {
- if (th instanceof IOException) throw (IOException) th;
- if (th instanceof RuntimeException) throw (RuntimeException) th;
- if (th instanceof Error) throw (Error) th;
- throw new RuntimeException(th);
- }
+ reThrow(th);
}
/**
@@ -238,7 +222,7 @@ public final class IOUtils {
* @param exception this exception should get the suppressed one added
* @param suppressed the suppressed exception
*/
- private static final void addSuppressed(Throwable exception, Throwable suppressed) {
+ private static void addSuppressed(Throwable exception, Throwable suppressed) {
if (SUPPRESS_METHOD != null && exception != null && suppressed != null) {
try {
SUPPRESS_METHOD.invoke(exception, suppressed);
@@ -361,4 +345,25 @@ public final class IOUtils {
close(fis, fos);
}
}
+
+ /**
+ * Simple utilty method that takes a previously caught
+ * {@code Throwable} and rethrows either {@code
+ * IOException} or an unchecked exception. If the
+ * argument is null then this method does nothing.
+ */
+ public static void reThrow(Throwable th) throws IOException {
+ if (th != null) {
+ if (th instanceof IOException) {
+ throw (IOException) th;
+ }
+ if (th instanceof RuntimeException) {
+ throw (RuntimeException) th;
+ }
+ if (th instanceof Error) {
+ throw (Error) th;
+ }
+ throw new RuntimeException(th);
+ }
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Oct 8 20:38:57 2013
@@ -1029,13 +1029,23 @@ public class TestIndexWriter extends Luc
@Override
public void run() {
// LUCENE-2239: won't work with NIOFS/MMAP
- Directory dir = new MockDirectoryWrapper(random, new RAMDirectory());
+ MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory());
+
+ // When interrupt arrives in w.close(), when it's
+ // writing liveDocs, this can lead to double-write of
+ // _X_N.del:
+ //dir.setPreventDoubleWrite(false);
IndexWriter w = null;
while(!finish) {
try {
while(!finish) {
if (w != null) {
+ // If interrupt arrives inside here, it's
+ // fine: we will cycle back and the first
+ // thing we do is try to close again,
+ // i.e. we'll never try to open a new writer
+ // until this one successfully closes:
w.close();
w = null;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue Oct 8 20:38:57 2013
@@ -40,6 +40,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
@@ -322,7 +323,7 @@ public class TestIndexWriterDelete exten
int value = 100;
try {
latch.await();
- for (int i = 0; i < 1000; i++) {
+ for (int j = 0; j < 1000; j++) {
Document doc = new Document();
doc.add(newTextField("content", "aaa", Field.Store.NO));
doc.add(newStringField("id", String.valueOf(id++), Field.Store.YES));
@@ -1228,13 +1229,11 @@ public class TestIndexWriterDelete exten
d.close();
}
- private static class FakeIOException extends IOException {
- }
-
- // Make sure if we hit disk full, and then later disk
- // frees up, and we successfully close IW or open an NRT
+ // Make sure if we hit a transient IOException (e.g., disk
+ // full), and then the exception stops (e.g., disk frees
+ // up), so we successfully close IW or open an NRT
// reader, we don't lose any deletes:
- public void testNoLostDeletesOnDiskFull() throws Exception {
+ public void testNoLostDeletesOnIOException() throws Exception {
int deleteCount = 0;
int docBase = 0;
@@ -1247,35 +1246,69 @@ public class TestIndexWriterDelete exten
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
StackTraceElement[] trace = new Exception().getStackTrace();
- if (shouldFail.get()) {
- for (int i = 0; i < trace.length; i++) {
- if ("writeLiveDocs".equals(trace[i].getMethodName())) {
- // Only sometimes throw the exc, so we get
- // it sometimes on creating the file, on
- // flushing buffer, on closing the file:
- if (random().nextInt(3) == 2) {
- if (VERBOSE) {
- System.out.println("TEST: now fail; exc:");
- new Throwable().printStackTrace(System.out);
- }
- shouldFail.set(false);
- throw new FakeIOException();
- } else {
- break;
- }
- }
+ if (shouldFail.get() == false) {
+ return;
+ }
+
+ boolean sawSeal = false;
+ boolean sawWrite = false;
+ for (int i = 0; i < trace.length; i++) {
+ if ("sealFlushedSegment".equals(trace[i].getMethodName())) {
+ sawSeal = true;
+ break;
+ }
+ if ("writeLiveDocs".equals(trace[i].getMethodName())) {
+ sawWrite = true;
}
}
+
+ // Don't throw exc if we are "flushing", else
+ // the segment is aborted and docs are lost:
+ if (sawWrite && sawSeal == false && random().nextInt(3) == 2) {
+ // Only sometimes throw the exc, so we get
+ // it sometimes on creating the file, on
+ // flushing buffer, on closing the file:
+ if (VERBOSE) {
+ System.out.println("TEST: now fail; thread=" + Thread.currentThread().getName() + " exc:");
+ new Throwable().printStackTrace(System.out);
+ }
+ shouldFail.set(false);
+ throw new FakeIOException();
+ }
}
});
+ RandomIndexWriter w = null;
+
for(int iter=0;iter<10*RANDOM_MULTIPLIER;iter++) {
int numDocs = atLeast(100);
if (VERBOSE) {
- System.out.println("\nTEST: iter=" + iter + " numDocs=" + numDocs + " docBase=" + docBase);
+ System.out.println("\nTEST: iter=" + iter + " numDocs=" + numDocs + " docBase=" + docBase + " delCount=" + deleteCount);
+ }
+ if (w == null) {
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ final MergeScheduler ms = iwc.getMergeScheduler();
+ if (ms instanceof ConcurrentMergeScheduler) {
+ final ConcurrentMergeScheduler suppressFakeIOE = new ConcurrentMergeScheduler() {
+ @Override
+ protected void handleMergeException(Throwable exc) {
+ // suppress only FakeIOException:
+ if (!(exc instanceof FakeIOException)) {
+ super.handleMergeException(exc);
+ }
+ }
+ };
+ final ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) ms;
+ suppressFakeIOE.setMaxMergesAndThreads(cms.getMaxMergeCount(), cms.getMaxThreadCount());
+ suppressFakeIOE.setMergeThreadPriority(cms.getMergeThreadPriority());
+ iwc.setMergeScheduler(suppressFakeIOE);
+ }
+ w = new RandomIndexWriter(random(), dir, iwc);
+ // Since we hit exc during merging, a partial
+ // forceMerge can easily return when there are still
+ // too many segments in the index:
+ w.setDoRandomForceMergeAssert(false);
}
- IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- IndexWriter w = new IndexWriter(dir, iwc);
for(int i=0;i<numDocs;i++) {
Document doc = new Document();
doc.add(new StringField("id", ""+(docBase+i), Field.Store.NO));
@@ -1283,9 +1316,8 @@ public class TestIndexWriterDelete exten
}
docCount += numDocs;
- IndexReader r = w.getReader();
- assertEquals(docCount-deleteCount, r.numDocs());
- r.close();
+ // TODO: we could make the test more evil, by letting
+ // it throw more than one exc, randomly, before "recovering"
// TODO: we could also install an infoStream and try
// to fail in "more evil" places inside BDS
@@ -1296,29 +1328,74 @@ public class TestIndexWriterDelete exten
for(int i=0;i<numDocs;i++) {
if (random().nextInt(10) == 7) {
+ if (VERBOSE) {
+ System.out.println(" delete id=" + (docBase+i));
+ }
deleteCount++;
w.deleteDocuments(new Term("id", ""+(docBase+i)));
}
}
- w.close();
+ // Trigger writeLiveDocs so we hit fake exc:
+ IndexReader r = w.getReader(true);
+
+ // Sometimes we will make it here (we only randomly
+ // throw the exc):
+ assertEquals(docCount-deleteCount, r.numDocs());
+ r.close();
+
+ // TODO: also call w.close() in here, sometimes,
+ // so we sometimes get a fail via dropAll
+
} catch (FakeIOException ioe) {
// expected
if (VERBOSE) {
System.out.println("TEST: w.close() hit expected IOE");
}
// No exception should happen here (we only fail once):
- w.close();
}
shouldFail.set(false);
- r = DirectoryReader.open(dir);
+ IndexReader r;
+
+ if (random().nextBoolean()) {
+ // Open non-NRT reader, to make sure the "on
+ // disk" bits are good:
+ if (VERBOSE) {
+ System.out.println("TEST: verify against non-NRT reader");
+ }
+ w.commit();
+ r = DirectoryReader.open(dir);
+ } else {
+ if (VERBOSE) {
+ System.out.println("TEST: verify against NRT reader");
+ }
+ r = w.getReader();
+ }
assertEquals(docCount-deleteCount, r.numDocs());
r.close();
+ // Sometimes re-use RIW, other times open new one:
+ if (random().nextBoolean()) {
+ if (VERBOSE) {
+ System.out.println("TEST: close writer");
+ }
+ w.close();
+ w = null;
+ }
+
docBase += numDocs;
}
+ if (w != null) {
+ w.close();
+ }
+
+ // Final verify:
+ IndexReader r = DirectoryReader.open(dir);
+ assertEquals(docCount-deleteCount, r.numDocs());
+ r.close();
+
dir.close();
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Tue Oct 8 20:38:57 2013
@@ -37,6 +37,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
@@ -159,6 +160,7 @@ public class TestIndexWriterReader exten
assertEquals(1, count(new Term("id", Integer.toString(8000)), r2));
r1.close();
+ assertTrue(r2.isCurrent());
writer.close();
assertTrue(r2.isCurrent());
@@ -1076,10 +1078,6 @@ public class TestIndexWriterReader exten
d.close();
}
- private static final class FakeIOException extends IOException {
- public FakeIOException() {}
- }
-
@Test
public void testNRTOpenExceptions() throws Exception {
// LUCENE-5262: test that several failed attempts to obtain an NRT reader
Modified: lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/DrillDownQueryTest.java Tue Oct 8 20:38:57 2013
@@ -66,6 +66,10 @@ public class DrillDownQueryTest extends
@AfterClass
public static void afterClassDrillDownQueryTest() throws Exception {
IOUtils.close(reader, taxo, dir, taxoDir);
+ reader = null;
+ taxo = null;
+ dir = null;
+ taxoDir = null;
}
@BeforeClass
Modified: lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java Tue Oct 8 20:38:57 2013
@@ -29,8 +29,8 @@ import java.util.Map;
import java.util.TimeZone;
import org.apache.lucene.analysis.*;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException;
@@ -48,6 +48,8 @@ import org.apache.lucene.search.Wildcard
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.automaton.BasicAutomata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
/**
* <p>
@@ -64,7 +66,17 @@ import org.apache.lucene.util.automaton.
//that it adjusts to fit the precedence requirement, adding its extra tests.
public class TestPrecedenceQueryParser extends LuceneTestCase {
- public static Analyzer qpAnalyzer = new QPTestAnalyzer();
+ public static Analyzer qpAnalyzer;
+
+ @BeforeClass
+ public static void beforeClass() {
+ qpAnalyzer = new QPTestAnalyzer();
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ qpAnalyzer = null;
+ }
public static final class QPTestFilter extends TokenFilter {
/**
Modified: lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java (original)
+++ lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestNumericQueryParser.java Tue Oct 8 20:38:57 2013
@@ -525,6 +525,7 @@ public class TestNumericQueryParser exte
reader = null;
directory.close();
directory = null;
+ qp = null;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java (original)
+++ lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java Tue Oct 8 20:38:57 2013
@@ -70,6 +70,8 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.automaton.BasicAutomata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.RegExp;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Ignore;
/**
@@ -81,7 +83,17 @@ import org.junit.Ignore;
// TODO: really this should extend QueryParserTestBase too!
public class TestQPHelper extends LuceneTestCase {
- public static Analyzer qpAnalyzer = new QPTestAnalyzer();
+ public static Analyzer qpAnalyzer;
+
+ @BeforeClass
+ public static void beforeClass() {
+ qpAnalyzer = new QPTestAnalyzer();
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ qpAnalyzer = null;
+ }
public static final class QPTestFilter extends TokenFilter {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
Modified: lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java Tue Oct 8 20:38:57 2013
@@ -41,8 +41,8 @@ import org.apache.lucene.index.Term;
//import org.apache.lucene.queryparser.classic.ParseException;
//import org.apache.lucene.queryparser.classic.QueryParser;
//import org.apache.lucene.queryparser.classic.QueryParserBase;
-//import org.apache.lucene.queryparser.classic.QueryParserTokenManager;
import org.apache.lucene.queryparser.classic.QueryParserBase;
+//import org.apache.lucene.queryparser.classic.QueryParserTokenManager;
import org.apache.lucene.queryparser.flexible.standard.CommonQueryParserConfiguration;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -51,6 +51,8 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.automaton.BasicAutomata;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.RegExp;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
/**
* Base Test class for QueryParser subclasses
@@ -59,8 +61,18 @@ import org.apache.lucene.util.automaton.
// to the core QP and subclass/use the parts that are not in the flexible QP
public abstract class QueryParserTestBase extends LuceneTestCase {
- public static Analyzer qpAnalyzer = new QPTestAnalyzer();
+ public static Analyzer qpAnalyzer;
+
+ @BeforeClass
+ public static void beforeClass() {
+ qpAnalyzer = new QPTestAnalyzer();
+ }
+ @AfterClass
+ public static void afterClass() {
+ qpAnalyzer = null;
+ }
+
public static final class QPTestFilter extends TokenFilter {
CharTermAttribute termAtt;
OffsetAttribute offsetAtt;
@@ -102,7 +114,6 @@ public abstract class QueryParserTestBas
}
}
-
public static final class QPTestAnalyzer extends Analyzer {
/** Filters MockTokenizer with StopFilter. */
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Tue Oct 8 20:38:57 2013
@@ -986,4 +986,9 @@ public class MockDirectoryWrapper extend
}
}
}
+
+ /** Use this when throwing fake {@code IOException},
+ * e.g. from {@link MockDirectoryWrapper.Failure}. */
+ public static class FakeIOException extends IOException {
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java?rev=1530416&r1=1530415&r2=1530416&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java Tue Oct 8 20:38:57 2013
@@ -108,11 +108,13 @@ public class ThrottledIndexOutput extend
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
final long before = System.nanoTime();
+ // TODO: sometimes, write only half the bytes, then
+ // sleep, then 2nd half, then sleep, so we sometimes
+ // interrupt having only written not all bytes
delegate.writeBytes(b, offset, length);
timeElapsed += System.nanoTime() - before;
pendingBytes += length;
sleep(getDelay(false));
-
}
protected long getDelay(boolean closing) {