You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2013/03/22 18:07:11 UTC
svn commit: r1459903 - in /lucene/dev/trunk/lucene: CHANGES.txt
core/src/java/org/apache/lucene/index/DirectoryReader.java
core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
Author: simonw
Date: Fri Mar 22 17:07:10 2013
New Revision: 1459903
URL: http://svn.apache.org/r1459903
Log:
LUCENE-4870: Lucene deletes entire index if and exception is thrown due do TooManyOpenFiles and OpenMode.CREATE_OR_APPEND
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1459903&r1=1459902&r2=1459903&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Mar 22 17:07:10 2013
@@ -191,6 +191,11 @@ Bug Fixes
* SOLR-4589: Fixed CPU spikes and poor performance in lazy field loading
of multivalued fields. (hossman)
+* LUCENE-4870: Fix bug where an entire index might be deleted by the IndexWriter
+ due to false detection if an index exists in the directory when
+ OpenMode.CREATE_OR_APPEND is used. This might also affect application that set
+ the open mode manually using DirectoryReader#indexExists. (Simon Willnauer)
+
Optimizations
* LUCENE-4819: Added Sorted[Set]DocValues.termsEnum(), and optimized the
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1459903&r1=1459902&r2=1459903&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java Fri Mar 22 17:07:10 2013
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.lucene.index.SegmentInfos.FindSegmentsFile;
import org.apache.lucene.search.SearcherManager; // javadocs
import org.apache.lucene.store.Directory;
@@ -319,7 +320,24 @@ public abstract class DirectoryReader ex
*/
public static boolean indexExists(Directory directory) {
try {
- new SegmentInfos().read(directory);
+ new FindSegmentsFile(directory) {
+ @Override
+ protected Object doBody(String segmentFileName) throws IOException {
+ try {
+ new SegmentInfos().read(directory, segmentFileName);
+ } catch (FileNotFoundException ex) {
+ if (!directory.fileExists(segmentFileName)) {
+ throw ex;
+ }
+ /* this is ok - we might have run into a access exception here.
+ * or even worse like on LUCENE-4870 this is triggered due to
+ * too many open files on the system. In that case we rather report
+ * a false positive here since wrongly returning false from indexExist
+ * can cause data loss since IW relies on this.*/
+ }
+ return null;
+ }
+ }.run();
return true;
} catch (IOException ioe) {
return false;
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1459903&r1=1459902&r2=1459903&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Fri Mar 22 17:07:10 2013
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
@@ -1627,4 +1628,75 @@ public class TestIndexWriterExceptions e
iw.close();
dir.close();
}
+
+ // See LUCENE-4870 TooManyOpenFiles errors are thrown as
+ // FNFExceptions which can trigger data loss.
+ public void testTooManyFileException() throws Exception {
+
+ // Create failure that throws Too many open files exception randomly
+ MockDirectoryWrapper.Failure failure = new MockDirectoryWrapper.Failure() {
+
+ @Override
+ public MockDirectoryWrapper.Failure reset() {
+ doFail = false;
+ return this;
+ }
+
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ if (doFail) {
+ if (random().nextBoolean()) {
+ throw new FileNotFoundException("some/file/name.ext (Too many open files)");
+ }
+ }
+ }
+ };
+
+ MockDirectoryWrapper dir = newMockDirectory();
+ // The exception is only thrown on open input
+ dir.setFailOnOpenInput(true);
+ dir.failOn(failure);
+
+ // Create an index with one document
+ IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter iw = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new StringField("foo", "bar", Field.Store.NO));
+ iw.addDocument(doc); // add a document
+ iw.commit();
+ DirectoryReader ir = DirectoryReader.open(dir);
+ assertEquals(1, ir.numDocs());
+ ir.close();
+ iw.close();
+
+ // Open and close the index a few times
+ for (int i = 0; i < 10; i++) {
+ failure.setDoFail();
+ iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ try {
+ iw = new IndexWriter(dir, iwc);
+ } catch (CorruptIndexException ex) {
+ // Exceptions are fine - we are running out of file handlers here
+ continue;
+ } catch (FileNotFoundException ex) {
+ continue;
+ }
+ failure.clearDoFail();
+ iw.close();
+ ir = DirectoryReader.open(dir);
+ assertEquals("lost document after iteration: " + i, 1, ir.numDocs());
+ ir.close();
+ }
+
+ // Check if document is still there
+ failure.clearDoFail();
+ ir = DirectoryReader.open(dir);
+ assertEquals(1, ir.numDocs());
+ ir.close();
+
+ dir.close();
+ }
+
+
+
}