You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/02/06 14:27:06 UTC
[13/17] lucene-solr git commit: improve tests;
fix RAFDirectory.openInput to respect pending delete
improve tests; fix RAFDirectory.openInput to respect pending delete
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/24f55abf
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/24f55abf
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/24f55abf
Branch: refs/heads/master
Commit: 24f55abfd508d4d36aa697c77fcded534016ca1c
Parents: 85c546b
Author: Mike McCandless <mi...@apache.org>
Authored: Fri Feb 5 12:19:24 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Fri Feb 5 12:19:24 2016 -0500
----------------------------------------------------------------------
.../src/java/org/apache/lucene/index/IndexWriter.java | 5 ++---
.../src/java/org/apache/lucene/store/FSDirectory.java | 6 ++++--
.../test/org/apache/lucene/index/TestIndexWriter.java | 2 +-
.../index/TestIndexWriterOutOfFileDescriptors.java | 2 ++
.../org/apache/lucene/index/TestStressIndexing2.java | 6 +++---
.../src/test/org/apache/lucene/index/TestStressNRT.java | 2 +-
.../src/java/org/apache/lucene/store/RAFDirectory.java | 1 +
.../java/org/apache/lucene/mockfile/VirusCheckingFS.java | 7 ++++---
.../org/apache/lucene/store/BaseDirectoryTestCase.java | 11 +++++++++--
.../org/apache/lucene/store/BaseLockFactoryTestCase.java | 2 ++
10 files changed, 29 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 7d72603..bd0de48 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -754,9 +754,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* IO error
*/
public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException {
- Directory unwrapped = FilterDirectory.unwrap(d);
- if (unwrapped instanceof FSDirectory && ((FSDirectory) unwrapped).checkPendingDeletions()) {
- throw new IllegalArgumentException("Directory still has pending deleted files; cannot initialize IndexWriter");
+ if (d instanceof FSDirectory && ((FSDirectory) d).checkPendingDeletions()) {
+ throw new IllegalArgumentException("Directory " + d + " is still has pending deleted files; cannot initialize IndexWriter");
}
conf.setIndexWriter(this); // prevent reuse by other instances
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
index 38c8fdc..9d0cfa7 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
@@ -280,11 +280,11 @@ public abstract class FSDirectory extends BaseDirectory {
@Override
public void sync(Collection<String> names) throws IOException {
ensureOpen();
- maybeDeletePendingFiles();
for (String name : names) {
fsync(name);
}
+ maybeDeletePendingFiles();
}
@Override
@@ -293,11 +293,12 @@ public abstract class FSDirectory extends BaseDirectory {
if (pendingDeletes.contains(source)) {
throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved");
}
- maybeDeletePendingFiles();
+ pendingDeletes.remove(dest);
Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
// TODO: should we move directory fsync to a separate 'syncMetadata' method?
// for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns
IOUtils.fsync(directory, true);
+ maybeDeletePendingFiles();
}
@Override
@@ -327,6 +328,7 @@ public abstract class FSDirectory extends BaseDirectory {
throw new NoSuchFileException("file \"" + name + "\" is already pending delete");
}
privateDeleteFile(name);
+ maybeDeletePendingFiles();
}
/** Tries to delete any pending deleted files, and returns true if
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 1c3568a..d0008c9 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -2737,7 +2737,7 @@ public class TestIndexWriter extends LuceneTestCase {
try {
w = new IndexWriter(dir, iwc);
} catch (IllegalArgumentException iae) {
- assertEquals("Directory still has pending deleted files; cannot initialize IndexWriter", iae.getMessage());
+ assertTrue(iae.getMessage().contains("still has pending deleted files; cannot initialize IndexWriter"));
}
in.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java
index 30aac07..c4f39a8 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java
@@ -26,10 +26,12 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LineFileDocs;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.TestUtil;
+@SuppressFileSystems("WindowsFS")
public class TestIndexWriterOutOfFileDescriptors extends LuceneTestCase {
public void test() throws Exception {
MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("TestIndexWriterOutOfFileDescriptors"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
index 6326209..931eea6 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
@@ -48,7 +48,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
static int seed=0;
public void testRandomIWReader() throws Throwable {
- Directory dir = newDirectory();
+ Directory dir = newMaybeVirusCheckingDirectory();
// TODO: verify equals using IW.getReader
DocsAndWriter dw = indexRandomIWReader(5, 3, 100, dir);
@@ -61,8 +61,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
}
public void testRandom() throws Throwable {
- Directory dir1 = newDirectory();
- Directory dir2 = newDirectory();
+ Directory dir1 = newMaybeVirusCheckingDirectory();
+ Directory dir2 = newMaybeVirusCheckingDirectory();
// mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
boolean doReaderPooling = random().nextBoolean();
Map<String,Document> docs = indexRandom(5, 3, 100, dir1, doReaderPooling);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java b/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
index b6cc489..3956613 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestStressNRT.java
@@ -104,7 +104,7 @@ public class TestStressNRT extends LuceneTestCase {
List<Thread> threads = new ArrayList<>();
- Directory dir = newDirectory();
+ Directory dir = newMaybeVirusCheckingDirectory();
final RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())));
writer.setDoRandomForceMergeAssert(false);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java b/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
index 94be104..c014060 100644
--- a/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
+++ b/lucene/misc/src/java/org/apache/lucene/store/RAFDirectory.java
@@ -66,6 +66,7 @@ public class RAFDirectory extends FSDirectory {
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
+ ensureCanRead(name);
final File path = directory.resolve(name).toFile();
RandomAccessFile raf = new RandomAccessFile(path, "r");
return new RAFIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
index 5d14a3b..2c24d10 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
@@ -28,8 +28,9 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.LuceneTestCase;
/**
- * Acts like Windows, where random programs may open the files you just wrote in an unfriendly
- * way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc.
+ * Acts like a virus checker on Windows, where random programs may open the files you just wrote in an unfriendly
+ * way preventing deletion (e.g. not passing FILE_SHARE_DELETE) or renaming or overwriting etc. This is more evil
+ * than WindowsFS which just prevents deletion of files you still old open.
*/
public class VirusCheckingFS extends FilterFileSystemProvider {
@@ -75,5 +76,5 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
super.delete(path);
}
- // TODO: rename? createOutput? deleteIfExists?
+ // TODO: we could be more evil here, e.g. rename, createOutput, deleteIfExists
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
index 892da1e..fadd1d8 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
@@ -1282,8 +1282,15 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
// expected
}
- // write the file again
- try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+ if (random().nextBoolean()) {
+ try (IndexOutput out = fsDir.createOutput(fileName + "z", IOContext.DEFAULT)) {
+ }
+ // Make sure we can rename onto the deleted file:
+ fsDir.renameFile(fileName + "z", fileName);
+ } else {
+ // write the file again
+ try (IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT)) {
+ }
}
assertEquals(0, fsDir.fileLength(fileName));
assertTrue(Arrays.asList(fsDir.listAll()).contains(fileName));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/24f55abf/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
index 2b6a6e3..7677bad 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseLockFactoryTestCase.java
@@ -160,6 +160,8 @@ public abstract class BaseLockFactoryTestCase extends LuceneTestCase {
// no unexpected exceptions are raised:
public void testStressLocks() throws Exception {
Path tempPath = createTempDir();
+ assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));
+
Directory dir = getDirectory(tempPath);
// First create a 1 doc index: