You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/04/04 20:15:03 UTC
svn commit: r1584846 - in /lucene/dev/branches/lucene_solr_4_7: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/util/junitcompat/ lucene/test-fra...
Author: mikemccand
Date: Fri Apr 4 18:15:02 2014
New Revision: 1584846
URL: http://svn.apache.org/r1584846
Log:
LUCENE-5574: when closing an NRT reader, do not delete files if the original writer has since been closed
Modified:
lucene/dev/branches/lucene_solr_4_7/ (props changed)
lucene/dev/branches/lucene_solr_4_7/lucene/ (props changed)
lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt
lucene/dev/branches/lucene_solr_4_7/lucene/core/ (props changed)
lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java
lucene/dev/branches/lucene_solr_4_7/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_4_7/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
Modified: lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt?rev=1584846&r1=1584845&r2=1584846&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/CHANGES.txt Fri Apr 4 18:15:02 2014
@@ -3,6 +3,17 @@ Lucene Change Log
For more information on past and future Lucene versions, please see:
http://s.apache.org/luceneversions
+======================= Lucene 4.7.2 =======================
+
+Bug Fixes
+
+* LUCENE-5574: Closing a near-real-time reader no longer attempts to
+ delete unreferenced files if the original writer has been closed;
+ this could cause index corruption in certain cases where index files
+ were directly changed (deleted, overwritten, etc.) in the index
+ directory outside of Lucene. (Simon Willnauer, Shai Erera, Robert
+ Muir, Mike McCandless)
+
======================= Lucene 4.7.1 =======================
Changes in Runtime Behavior
Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1584846&r1=1584845&r2=1584846&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Apr 4 18:15:02 2014
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.CollectionUtil;
@@ -261,6 +262,14 @@ final class IndexFileDeleter implements
deleteCommits();
}
+ private void ensureOpen() throws AlreadyClosedException {
+ if (writer == null) {
+ throw new AlreadyClosedException("this IndexWriter is closed");
+ } else {
+ writer.ensureOpen(false);
+ }
+ }
+
public SegmentInfos getLastSegmentInfos() {
return lastSegmentInfos;
}
@@ -577,6 +586,7 @@ final class IndexFileDeleter implements
void deleteFile(String fileName)
throws IOException {
assert locked();
+ ensureOpen();
try {
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "delete \"" + fileName + "\"");
Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1584846&r1=1584845&r2=1584846&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Apr 4 18:15:02 2014
@@ -4644,8 +4644,7 @@ public class IndexWriter implements Clos
deleter.revisitPolicy();
}
- // Called by DirectoryReader.doClose
- synchronized void deletePendingFiles() throws IOException {
+ private synchronized void deletePendingFiles() throws IOException {
deleter.deletePendingFiles();
}
@@ -4744,10 +4743,12 @@ public class IndexWriter implements Clos
}
synchronized void incRefDeleter(SegmentInfos segmentInfos) throws IOException {
+ ensureOpen();
deleter.incRef(segmentInfos, false);
}
synchronized void decRefDeleter(SegmentInfos segmentInfos) throws IOException {
+ ensureOpen();
deleter.decRef(segmentInfos);
}
Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java?rev=1584846&r1=1584845&r2=1584846&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java Fri Apr 4 18:15:02 2014
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
@@ -369,11 +370,15 @@ final class StandardDirectoryReader exte
}
if (writer != null) {
- writer.decRefDeleter(segmentInfos);
-
- // Since we just closed, writer may now be able to
- // delete unused files:
- writer.deletePendingFiles();
+ try {
+ writer.decRefDeleter(segmentInfos);
+ } catch (AlreadyClosedException ex) {
+ // This is OK, it just means our original writer was
+ // closed before we were, and this may leave some
+ // un-referenced files in the index, which is
+ // harmless. The next time IW is opened on the
+ // index, it will delete them.
+ }
}
// throw the first exception
Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1584846&r1=1584845&r2=1584846&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Apr 4 18:15:02 2014
@@ -2411,4 +2411,42 @@ public class TestIndexWriter extends Luc
r.close();
dir.close();
}
+
+ // LUCENE-5574
+ public void testClosingNRTReaderDoesNotCorruptYourIndex() throws IOException {
+ MockDirectoryWrapper dir = newMockDirectory();
+
+ // Allow deletion of still open files:
+ dir.setNoDeleteOpenFile(false);
+
+ // Allow writing to same file more than once:
+ dir.setPreventDoubleWrite(false);
+
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ LogMergePolicy lmp = new LogDocMergePolicy();
+ lmp.setMergeFactor(2);
+ iwc.setMergePolicy(lmp);
+
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ doc.add(new TextField("a", "foo", Field.Store.NO));
+ w.addDocument(doc);
+ w.commit();
+ w.addDocument(doc);
+
+ // Get a new reader, but this also sets off a merge:
+ IndexReader r = w.getReader();
+ w.close();
+
+ // Blow away index and make a new writer:
+ for(String fileName : dir.listAll()) {
+ dir.deleteFile(fileName);
+ }
+
+ w = new RandomIndexWriter(random(), dir);
+ w.addDocument(doc);
+ w.close();
+ r.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java?rev=1584846&r1=1584845&r2=1584846&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java (original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java Fri Apr 4 18:15:02 2014
@@ -22,15 +22,14 @@ import java.util.Collections;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.JUnitCore;
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
-
import com.carrotsearch.randomizedtesting.RandomizedTest;
// LUCENE-4456: Test that we fail if there are unreferenced files
@@ -41,7 +40,8 @@ public class TestFailIfUnreferencedFiles
public static class Nested1 extends WithNestedTests.AbstractNestedTest {
public void testDummy() throws Exception {
- Directory dir = newMockDirectory();
+ MockDirectoryWrapper dir = newMockDirectory();
+ dir.setAssertNoUnrefencedFilesOnClose(true);
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
iw.addDocument(new Document());
iw.close();
Modified: lucene/dev/branches/lucene_solr_4_7/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_7/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1584846&r1=1584845&r2=1584846&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_7/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene_solr_4_7/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Fri Apr 4 18:15:02 2014
@@ -605,7 +605,8 @@ public class MockDirectoryWrapper extend
return size;
}
- private boolean assertNoUnreferencedFilesOnClose = true;
+ // NOTE: This is off by default; see LUCENE-5574
+ private boolean assertNoUnreferencedFilesOnClose;
public void setAssertNoUnrefencedFilesOnClose(boolean v) {
assertNoUnreferencedFilesOnClose = v;