You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/02/03 10:23:18 UTC
[5/5] lucene-solr git commit: fix a few tests; add some nocommits
fix a few tests; add some nocommits
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b4a2bf2b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b4a2bf2b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b4a2bf2b
Branch: refs/heads/lucene-6835
Commit: b4a2bf2b41952be6ec3027943425d15365bcd9a0
Parents: 84f4458
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Feb 2 18:07:06 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Feb 2 18:07:06 2016 -0500
----------------------------------------------------------------------
.../java/org/apache/lucene/store/Directory.java | 1 +
.../org/apache/lucene/store/FSDirectory.java | 55 ++++++++++++++------
.../lucene/index/TestDirectoryReader.java | 6 ++-
.../org/apache/lucene/util/TestIOUtils.java | 3 ++
.../apache/lucene/mockfile/VirusCheckingFS.java | 4 ++
5 files changed, 52 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/java/org/apache/lucene/store/Directory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java
index b9e5ad4..bb12c92 100644
--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java
@@ -47,6 +47,7 @@ public abstract class Directory implements Closeable {
*
* @throws IOException in case of IO error
*/
+ // nocommit should this sort?
public abstract String[] listAll() throws IOException;
/** Removes the specified files from the directory. If an exception is thrown, behavior is undefined
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
index 0e1d4e9..9eb78b4 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
@@ -229,22 +229,20 @@ public abstract class FSDirectory extends BaseDirectory {
return listAll(directory, pendingDeletes);
}
- /** Returns the length in bytes of a file in the directory. */
@Override
public long fileLength(String name) throws IOException {
ensureOpen();
return Files.size(directory.resolve(name));
}
- /** Removes an existing file in the directory. */
@Override
public void deleteFiles(Collection<String> names) throws IOException {
ensureOpen();
+ // nocommit isn't it an error if they were already pending delete?
pendingDeletes.addAll(names);
deletePendingFiles();
}
- /** Creates an IndexOutput for the file with the given name. */
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
@@ -277,7 +275,7 @@ public abstract class FSDirectory extends BaseDirectory {
protected void ensureCanRead(String name) throws IOException {
deletePendingFiles();
if (pendingDeletes.contains(name)) {
- throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be overwritten");
+ throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be opened for read");
}
}
@@ -299,7 +297,6 @@ public abstract class FSDirectory extends BaseDirectory {
IOUtils.fsync(directory, true);
}
- /** Closes the store to future operations. */
@Override
public synchronized void close() throws IOException {
isOpen = false;
@@ -312,7 +309,6 @@ public abstract class FSDirectory extends BaseDirectory {
return directory;
}
- /** For debug output. */
@Override
public String toString() {
return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
@@ -324,14 +320,13 @@ public abstract class FSDirectory extends BaseDirectory {
}
/** Returns true if the file was successfully removed. */
- private boolean deleteFile(String name) throws IOException {
+ private synchronized boolean deleteFile(String name) throws IOException {
+ pendingDeletes.remove(name);
try {
Files.delete(directory.resolve(name));
- pendingDeletes.remove(name);
return true;
} catch (NoSuchFileException | FileNotFoundException e) {
// We were asked to delete a non-existent file:
- pendingDeletes.remove(name);
throw e;
} catch (IOException ioe) {
// On windows, a file delete can fail because there's still an open
@@ -358,21 +353,36 @@ public abstract class FSDirectory extends BaseDirectory {
}
/** Try to delete any pending files that we had previously tried to delete but failed
- * because we are on Windows and the files were still
- * held open. */
- public void deletePendingFiles() throws IOException {
+ * because we are on Windows and the files were still held open. */
+ public synchronized void deletePendingFiles() throws IOException {
// TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are closed?
// Clone the set because it will change as we iterate:
List<String> toDelete = new ArrayList<>(pendingDeletes);
+ System.out.println("del pending: " + pendingDeletes);
// First pass: delete any segments_N files. We do these first to be certain stale commit points are removed
// before we remove any files they reference. If any delete of segments_N fails, we leave all other files
// undeleted so index is never in a corrupt state:
+ Throwable firstException = null;
for (String fileName : toDelete) {
if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
- if (deleteFile(fileName) == false) {
- return;
+ try {
+ if (deleteFile(fileName) == false) {
+ // nocommit
+ System.out.println(" false on " + fileName + "; skipping the rest");
+ return;
+ }
+ } catch (Throwable t) {
+ if (firstException == null) {
+ firstException = t;
+ } else {
+ firstException.addSuppressed(t);
+ }
+ // nocommit
+ System.out.println(" fail on " + fileName + ":");
+ t.printStackTrace(System.out);
+ throw t;
}
}
}
@@ -381,9 +391,24 @@ public abstract class FSDirectory extends BaseDirectory {
// leave a corrupt commit in the index even in the presense of virus checkers:
for(String fileName : toDelete) {
if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
- deleteFile(fileName);
+ try {
+ deleteFile(fileName);
+ } catch (Throwable t) {
+ if (firstException == null) {
+ firstException = t;
+ } else {
+ firstException.addSuppressed(t);
+ }
+ // nocommit
+ System.out.println(" fail on " + fileName + ":");
+ t.printStackTrace(System.out);
+ throw t;
+ }
}
}
+
+ // Does nothing if firstException is null:
+ IOUtils.reThrow(firstException);
}
final class FSIndexOutput extends OutputStreamIndexOutput {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index 2c4f392..2213033 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -472,7 +472,10 @@ void assertTermDocsCount(String msg,
public void testOpenReaderAfterDelete() throws IOException {
Path dirFile = createTempDir("deletetest");
Directory dir = newFSDirectory(dirFile);
- assumeFalse("test deletes files directly", TestUtil.hasVirusChecker(dir));
+ if (TestUtil.hasVirusChecker(dir)) {
+ dir.close();
+ assumeTrue("test deletes files directly", false);
+ }
if (dir instanceof BaseDirectoryWrapper) {
((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we will hit NoSuchFileException in MDW since we nuked it!
}
@@ -1055,7 +1058,6 @@ void assertTermDocsCount(String msg,
public void testIndexExistsOnNonExistentDirectory() throws Exception {
Path tempDir = createTempDir("testIndexExistsOnNonExistentDirectory");
- IOUtils.rm(tempDir);
Directory dir = newFSDirectory(tempDir);
assertFalse(DirectoryReader.indexExists(dir));
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
index 7a112cd..726f4f8 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestIOUtils.java
@@ -38,8 +38,11 @@ import java.util.UUID;
import org.apache.lucene.mockfile.FilterFileSystem;
import org.apache.lucene.mockfile.FilterFileSystemProvider;
import org.apache.lucene.mockfile.FilterPath;
+import org.apache.lucene.mockfile.VirusCheckingFS;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
/** Simple test methods for IOUtils */
+@SuppressFileSystems("VirusCheckingFS")
public class TestIOUtils extends LuceneTestCase {
public void testDeleteFileIgnoringExceptions() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a2bf2b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
index 801a688..b30ce94 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/VirusCheckingFS.java
@@ -25,6 +25,7 @@ import java.nio.file.Path;
import java.util.Random;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.util.LuceneTestCase;
/**
* Acts like Windows, where random programs may open the files you just wrote in an unfriendly
@@ -56,6 +57,9 @@ public class VirusCheckingFS extends FilterFileSystemProvider {
&& Files.exists(path) // important that we NOT delay a NoSuchFileException until later
&& path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false // life is particularly difficult if the virus checker hits our lock file
&& random.nextInt(5) == 1) {
+ if (true || LuceneTestCase.VERBOSE) {
+ System.out.println("NOTE: VirusCheckingFS now refusing to delete " + path);
+ }
throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file \"" + path + "\"");
}
super.delete(path);