You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/10/12 22:24:08 UTC
svn commit: r1708230 - in /lucene/dev/branches/lucene6835/lucene:
codecs/src/java/org/apache/lucene/codecs/simpletext/
core/src/java/org/apache/lucene/codecs/lucene50/
core/src/java/org/apache/lucene/index/
core/src/java/org/apache/lucene/store/ core/s...
Author: mikemccand
Date: Mon Oct 12 20:24:08 2015
New Revision: 1708230
URL: http://svn.apache.org/viewvc?rev=1708230&view=rev
Log:
LUCENE-6835: starting patch
Modified:
lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
Modified: lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java (original)
+++ lucene/dev/branches/lucene6835/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java Mon Oct 12 20:24:08 2015
@@ -145,7 +145,7 @@ public class SimpleTextCompoundFormat ex
public void sync(Collection<String> names) { throw new UnsupportedOperationException(); }
@Override
- public void deleteFile(String name) { throw new UnsupportedOperationException(); }
+ public void deleteFiles(Collection<String> name) { throw new UnsupportedOperationException(); }
@Override
public void renameFile(String source, String dest) { throw new UnsupportedOperationException(); }
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java Mon Oct 12 20:24:08 2015
@@ -17,6 +17,13 @@ package org.apache.lucene.codecs.lucene5
* limitations under the License.
*/
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFileNames;
@@ -29,13 +36,6 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.Lock;
import org.apache.lucene.util.IOUtils;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
/**
* Class for accessing a compound stream.
* This class implements a directory, but is limited to only read operations.
@@ -147,7 +147,7 @@ final class Lucene50CompoundReader exten
/** Not implemented
* @throws UnsupportedOperationException always: not supported by CFS */
@Override
- public void deleteFile(String name) {
+ public void deleteFiles(Collection<String> name) {
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Mon Oct 12 20:24:08 2015
@@ -78,11 +78,6 @@ import java.util.regex.Matcher;
final class IndexFileDeleter implements Closeable {
- /* Files that we tried to delete but failed (likely
- * because they are open and we are running on Windows),
- * so we will retry them again later: */
- private final Set<String> deletable = new HashSet<>();
-
/* Reference count for all files in the index.
* Counts how many existing commits reference a file.
**/
@@ -219,6 +214,7 @@ final class IndexFileDeleter implements
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
+ Set<String> toDelete = new HashSet<>();
for(Map.Entry<String, RefCount> entry : refCounts.entrySet() ) {
RefCount rc = entry.getValue();
final String fileName = entry.getKey();
@@ -230,10 +226,12 @@ final class IndexFileDeleter implements
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "init: removing unreferenced file \"" + fileName + "\"");
}
- deleteFile(fileName);
+ toDelete.add(fileName);
}
}
+ deleteFiles(toDelete);
+
// Finally, give policy a chance to remove things on
// startup:
policy.onInit(commits);
@@ -419,7 +417,7 @@ final class IndexFileDeleter implements
*/
void refresh() throws IOException {
assert locked();
- deletable.clear();
+ Set<String> toDelete = new HashSet<>();
String[] files = directory.listAll();
@@ -439,15 +437,15 @@ final class IndexFileDeleter implements
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "refresh: removing newly created unreferenced file \"" + fileName + "\"");
}
- deletable.add(fileName);
+ toDelete.add(fileName);
}
}
- deletePendingFiles();
+ deleteFiles(toDelete);
}
@Override
- public void close() {
+ public void close() throws IOException {
// DecRef old files from the last checkpoint, if any:
assert locked();
@@ -458,8 +456,6 @@ final class IndexFileDeleter implements
lastFiles.clear();
}
}
-
- deletePendingFiles();
}
/**
@@ -483,39 +479,6 @@ final class IndexFileDeleter implements
}
}
- public void deletePendingFiles() {
- assert locked();
-
- // Clone the set because it will change as we iterate:
- List<String> toDelete = new ArrayList<>(deletable);
-
- // First pass: delete any segments_N files. We do these first to be certain stale commit points are removed
- // before we remove any files they reference. If any delete of segments_N fails, we leave all other files
- // undeleted so index is never in a corrupt state:
- for (String fileName : toDelete) {
- RefCount rc = refCounts.get(fileName);
- if (rc != null && rc.count > 0) {
- // LUCENE-5904: should never happen! This means we are about to pending-delete a referenced index file
- throw new IllegalStateException("file \"" + fileName + "\" is in pending delete set but has non-zero refCount=" + rc.count);
- } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
- if (deleteFile(fileName) == false) {
- if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "failed to remove commit point \"" + fileName + "\"; skipping deletion of all other pending files");
- }
- return;
- }
- }
- }
-
- // Only delete other files if we were able to remove the segments_N files; this way we never
- // leave a corrupt commit in the index even in the presense of virus checkers:
- for(String fileName : toDelete) {
- if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
- deleteFile(fileName);
- }
- }
- }
-
/**
* For definition of "check point" see IndexWriter comments:
* "Clarification: Check Points (and commits)".
@@ -604,12 +567,15 @@ final class IndexFileDeleter implements
}
/** Decrefs all provided files, even on exception; throws first exception hit, if any. */
- void decRef(Collection<String> files) {
+ void decRef(Collection<String> files) throws IOException {
assert locked();
+ Set<String> toDelete = new HashSet<>();
Throwable firstThrowable = null;
for(final String file : files) {
try {
- decRef(file);
+ if (decRef(file)) {
+ toDelete.add(file);
+ }
} catch (Throwable t) {
if (firstThrowable == null) {
// Save first exception and throw it in the end, but be sure to finish decRef all files
@@ -619,7 +585,7 @@ final class IndexFileDeleter implements
}
try {
- deletePendingFiles();
+ deleteFiles(toDelete);
} catch (Throwable t) {
if (firstThrowable == null) {
// Save first exception and throw it in the end, but be sure to finish decRef all files
@@ -628,27 +594,31 @@ final class IndexFileDeleter implements
}
// NOTE: does nothing if firstThrowable is null
- IOUtils.reThrowUnchecked(firstThrowable);
+ IOUtils.reThrow(firstThrowable);
}
/** Decrefs all provided files, ignoring any exceptions hit; call this if
* you are already handling an exception. */
void decRefWhileHandlingException(Collection<String> files) {
assert locked();
+ Set<String> toDelete = new HashSet<>();
for(final String file : files) {
try {
- decRef(file);
+ if (decRef(file)) {
+ toDelete.add(file);
+ }
} catch (Throwable t) {
}
}
try {
- deletePendingFiles();
+ deleteFiles(toDelete);
} catch (Throwable t) {
}
}
- private void decRef(String fileName) {
+ /** Returns true if the file should now be deleted. */
+ private boolean decRef(String fileName) {
assert locked();
RefCount rc = getRefCount(fileName);
if (infoStream.isEnabled("IFD")) {
@@ -656,14 +626,13 @@ final class IndexFileDeleter implements
infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
}
}
- if (0 == rc.DecRef()) {
+ if (rc.DecRef() == 0) {
// This file is no longer referenced by any past
// commit points nor by the in-memory SegmentInfos:
- try {
- deletable.add(fileName);
- } finally {
- refCounts.remove(fileName);
- }
+ refCounts.remove(fileName);
+ return true;
+ } else {
+ return false;
}
}
@@ -686,8 +655,6 @@ final class IndexFileDeleter implements
RefCount rc;
if (!refCounts.containsKey(fileName)) {
rc = new RefCount(fileName);
- // We should never incRef a file we are already wanting to delete:
- assert deletable == null || deletable.contains(fileName) == false: "file \"" + fileName + "\" cannot be incRef'd: it's already pending delete";
refCounts.put(fileName, rc);
} else {
rc = refCounts.get(fileName);
@@ -699,6 +666,7 @@ final class IndexFileDeleter implements
* (have not yet been incref'd). */
void deleteNewFiles(Collection<String> files) throws IOException {
assert locked();
+ Set<String> toDelete = new HashSet<>();
for (final String fileName: files) {
// NOTE: it's very unusual yet possible for the
// refCount to be present and 0: it can happen if you
@@ -710,45 +678,31 @@ final class IndexFileDeleter implements
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "will delete new file \"" + fileName + "\"");
}
- deletable.add(fileName);
+ toDelete.add(fileName);
}
}
- deletePendingFiles();
+ deleteFiles(toDelete);
}
- /** Returns true if the delete succeeded. Otherwise, the fileName is
- * added to the deletable set so we will retry the delete later, and
- * we return false. */
- private boolean deleteFile(String fileName) {
+ private void deleteFiles(Collection<String> names) throws IOException {
assert locked();
ensureOpen();
+ if (names.isEmpty()) {
+ return;
+ }
try {
if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "delete \"" + fileName + "\"");
+ infoStream.message("IFD", "delete \"" + names + "\"");
}
- directory.deleteFile(fileName);
- deletable.remove(fileName);
- return true;
- } catch (IOException e) { // if delete fails
-
+ directory.deleteFiles(names);
+ } catch (NoSuchFileException | FileNotFoundException e) { // if delete fails
// IndexWriter should only ask us to delete files it knows it wrote, so if we hit this, something is wrong!
- // LUCENE-6684: we suppress this assert for Windows, since a file could be in a confusing "pending delete" state:
- assert Constants.WINDOWS || e instanceof NoSuchFileException == false: "hit unexpected NoSuchFileException: file=" + fileName;
- assert Constants.WINDOWS || e instanceof FileNotFoundException == false: "hit unexpected FileNotFoundException: file=" + fileName;
-
- // Some operating systems (e.g. Windows) don't
- // permit a file to be deleted while it is opened
- // for read (e.g. by another process or thread). So
- // we assume that when a delete fails it is because
- // the file is open in another process, and queue
- // the file for subsequent deletion.
-
- if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "unable to remove file \"" + fileName + "\": " + e.toString() + "; Will re-try later.");
+ if (Constants.WINDOWS) {
+ // LUCENE-6684: we suppress this assert for Windows, since a file could be in a confusing "pending delete" state:
+ } else {
+ throw e;
}
- deletable.add(fileName);
- return false;
}
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Mon Oct 12 20:24:08 2015
@@ -3561,8 +3561,6 @@ public class IndexWriter implements Clos
}
}
- deleter.deletePendingFiles();
-
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "after commitMerge: " + segString());
}
@@ -4608,14 +4606,9 @@ public class IndexWriter implements Clos
*/
public synchronized void deleteUnusedFiles() throws IOException {
ensureOpen(false);
- deleter.deletePendingFiles();
deleter.revisitPolicy();
}
- private synchronized void deletePendingFiles() throws IOException {
- deleter.deletePendingFiles();
- }
-
/**
* NOTE: this method creates a compound file for all files returned by
* info.files(). While, generally, this may include separate norms and
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java Mon Oct 12 20:24:08 2015
@@ -212,7 +212,7 @@ public class PersistentSnapshotDeletionP
private synchronized void clearPriorSnapshots() throws IOException {
for(String file : dir.listAll()) {
if (file.startsWith(SNAPSHOTS_PREFIX)) {
- dir.deleteFile(file);
+ dir.deleteFiles(Collections.singleton(file));
}
}
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/Directory.java Mon Oct 12 20:24:08 2015
@@ -49,8 +49,8 @@ public abstract class Directory implemen
*/
public abstract String[] listAll() throws IOException;
- /** Removes an existing file in the directory. */
- public abstract void deleteFile(String name) throws IOException;
+ /** Removes the specified files from the directory. */
+ public abstract void deleteFiles(Collection<String> name) throws IOException;
/**
* Returns the length of a file in the directory. This method follows the
@@ -67,7 +67,6 @@ public abstract class Directory implemen
*/
public abstract long fileLength(String name) throws IOException;
-
/** Creates a new, empty file in the directory with the given name.
Returns a stream writing this file. */
public abstract IndexOutput createOutput(String name, IOContext context)
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java Mon Oct 12 20:24:08 2015
@@ -17,18 +17,25 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.FileNotFoundException;
import java.io.FilterOutputStream;
import java.io.IOException;
+import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
-import java.nio.channels.ClosedChannelException; // javadoc @link
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
@@ -115,6 +122,9 @@ public abstract class FSDirectory extend
protected final Path directory; // The underlying filesystem directory
+ /** Files we want to delete, but hit exception (on Windows) last time we tried. */
+ private final Set<String> pendingDeletes = Collections.newSetFromMap(new ConcurrentHashMap<String,Boolean>());
+
/** Create a new FSDirectory for the named location (ctor for subclasses).
* The directory is created at the named location if it does not yet exist.
*
@@ -211,17 +221,18 @@ public abstract class FSDirectory extend
/** Removes an existing file in the directory. */
@Override
- public void deleteFile(String name) throws IOException {
+ public void deleteFiles(Collection<String> names) throws IOException {
ensureOpen();
- Files.delete(directory.resolve(name));
+ pendingDeletes.addAll(names);
+ deletePendingFiles();
}
/** Creates an IndexOutput for the file with the given name. */
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
ensureOpen();
-
ensureCanWrite(name);
+ deletePendingFiles();
return new FSIndexOutput(name);
}
@@ -249,8 +260,9 @@ public abstract class FSDirectory extend
/** Closes the store to future operations. */
@Override
- public synchronized void close() {
+ public synchronized void close() throws IOException {
isOpen = false;
+ deletePendingFiles();
}
/** @return the underlying filesystem directory */
@@ -265,6 +277,63 @@ public abstract class FSDirectory extend
return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
}
+ protected void fsync(String name) throws IOException {
+ deletePendingFiles();
+ IOUtils.fsync(directory.resolve(name), false);
+ }
+
+ /** Returns true if the file was successfully removed. */
+ private boolean deleteFile(String name) throws IOException {
+ try {
+ Files.delete(directory.resolve(name));
+ pendingDeletes.remove(name);
+ return true;
+ } catch (NoSuchFileException | FileNotFoundException e) {
+ // We were asked to delete a non-existent file
+ throw e;
+ } catch (IOException ioe) {
+ // On windows, a file delete can fail because there's still an open
+ // file handle against it. We record this in pendingDeletes and
+ // try again later.
+
+ // TODO: this is hacky/lenient (we don't know which IOException this is), and
+ // it should only happen on filesystems that can do this, so really we should
+ // move this logic to WindowsDirectory or something
+
+ // nocommit can/should we do if (Constants.WINDOWS) here, else throw the exc?
+ // but what about a Linux box with a CIFS mount?
+ pendingDeletes.add(name);
+ return false;
+ }
+ }
+
+ // TODO: public?
+ // TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are closed?
+ private void deletePendingFiles() throws IOException {
+
+ // Clone the set because it will change as we iterate:
+ List<String> toDelete = new ArrayList<>(pendingDeletes);
+
+ // First pass: delete any segments_N files. We do these first to be certain stale commit points are removed
+ // before we remove any files they reference. If any delete of segments_N fails, we leave all other files
+ // undeleted so index is never in a corrupt state:
+ for (String fileName : toDelete) {
+ if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+ if (deleteFile(fileName) == false) {
+ return;
+ }
+ }
+ }
+
+ // Only delete other files if we were able to remove the segments_N files; this way we never
+ // leave a corrupt commit in the index even in the presense of virus checkers:
+ for(String fileName : toDelete) {
+ if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
+ deleteFile(fileName);
+ }
+ }
+ }
+
final class FSIndexOutput extends OutputStreamIndexOutput {
/**
* The maximum chunk size is 8192 bytes, because file channel mallocs
@@ -287,8 +356,4 @@ public abstract class FSDirectory extend
}, CHUNK_SIZE);
}
}
-
- protected void fsync(String name) throws IOException {
- IOUtils.fsync(directory.resolve(name), false);
- }
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java Mon Oct 12 20:24:08 2015
@@ -141,8 +141,21 @@ public class FileSwitchDirectory extends
}
@Override
- public void deleteFile(String name) throws IOException {
- getDirectory(name).deleteFile(name);
+ public void deleteFiles(Collection<String> names) throws IOException {
+ Set<String> primaryToDelete = new HashSet<>();
+ Set<String> secondaryToDelete = new HashSet<>();
+ for(String name : names) {
+ if (getDirectory(name) == primaryDir) {
+ primaryToDelete.add(name);
+ } else {
+ secondaryToDelete.add(name);
+ }
+ }
+ try {
+ primaryDir.deleteFiles(primaryToDelete);
+ } finally {
+ secondaryDir.deleteFiles(secondaryToDelete);
+ }
}
@Override
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/FilterDirectory.java Mon Oct 12 20:24:08 2015
@@ -58,8 +58,8 @@ public class FilterDirectory extends Dir
}
@Override
- public void deleteFile(String name) throws IOException {
- in.deleteFile(name);
+ public void deleteFiles(Collection<String> names) throws IOException {
+ in.deleteFiles(names);
}
@Override
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/LockValidatingDirectoryWrapper.java Mon Oct 12 20:24:08 2015
@@ -33,9 +33,9 @@ public final class LockValidatingDirecto
}
@Override
- public void deleteFile(String name) throws IOException {
+ public void deleteFiles(Collection<String> names) throws IOException {
writeLock.ensureValid();
- in.deleteFile(name);
+ in.deleteFiles(names);
}
@Override
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java Mon Oct 12 20:24:08 2015
@@ -108,14 +108,23 @@ public class NRTCachingDirectory extends
}
@Override
- public synchronized void deleteFile(String name) throws IOException {
+ public synchronized void deleteFiles(Collection<String> names) throws IOException {
if (VERBOSE) {
- System.out.println("nrtdir.deleteFile name=" + name);
+ System.out.println("nrtdir.deleteFiles names=" + names);
}
- if (cache.fileNameExists(name)) {
- cache.deleteFile(name);
- } else {
- in.deleteFile(name);
+ Set<String> cacheToDelete = new HashSet<>();
+ Set<String> toDelete = new HashSet<>();
+ for(String name : names) {
+ if (cache.fileNameExists(name)) {
+ cacheToDelete.add(name);
+ } else {
+ toDelete.add(name);
+ }
+ }
+ try {
+ cache.deleteFiles(cacheToDelete);
+ } finally {
+ in.deleteFiles(toDelete);
}
}
@@ -142,14 +151,14 @@ public class NRTCachingDirectory extends
System.out.println(" to cache");
}
try {
- in.deleteFile(name);
+ in.deleteFiles(Collections.singleton(name));
} catch (IOException ioe) {
// This is fine: file may not exist
}
return cache.createOutput(name, context);
} else {
try {
- cache.deleteFile(name);
+ cache.deleteFiles(Collections.singleton(name));
} catch (IOException ioe) {
// This is fine: file may not exist
}
@@ -259,7 +268,7 @@ public class NRTCachingDirectory extends
synchronized(this) {
// Must sync here because other sync methods have
// if (cache.fileNameExists(name)) { ... } else { ... }:
- cache.deleteFile(fileName);
+ cache.deleteFiles(Collections.singleton(fileName));
}
}
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java Mon Oct 12 20:24:08 2015
@@ -17,8 +17,8 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
@@ -154,14 +154,17 @@ public class RAMDirectory extends BaseDi
* @throws IOException if the file does not exist
*/
@Override
- public void deleteFile(String name) throws IOException {
+ public void deleteFiles(Collection<String> names) throws IOException {
ensureOpen();
- RAMFile file = fileMap.remove(name);
- if (file != null) {
- file.directory = null;
- sizeInBytes.addAndGet(-file.sizeInBytes);
- } else {
- throw new FileNotFoundException(name);
+ for(String name : names) {
+ RAMFile file = fileMap.remove(name);
+ if (file != null) {
+ file.directory = null;
+ sizeInBytes.addAndGet(-file.sizeInBytes);
+ } else {
+ // nocommit but should we keep trying to delete the rest, and throw this only at the end?
+ throw new FileNotFoundException(name);
+ }
}
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/store/TrackingDirectoryWrapper.java Mon Oct 12 20:24:08 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -33,9 +34,11 @@ public final class TrackingDirectoryWrap
}
@Override
- public void deleteFile(String name) throws IOException {
- in.deleteFile(name);
- createdFileNames.remove(name);
+ public void deleteFiles(Collection<String> names) throws IOException {
+ in.deleteFiles(names);
+ for(String name : names) {
+ createdFileNames.remove(name);
+ }
}
@Override
Modified: lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Mon Oct 12 20:24:08 2015
@@ -190,13 +190,11 @@ public final class IOUtils {
* <p>
* Note that the files should not be null.
*/
- public static void deleteFilesIgnoringExceptions(Directory dir, String... files) {
- for (String name : files) {
- try {
- dir.deleteFile(name);
- } catch (Throwable ignored) {
- // ignore
- }
+ public static void deleteFilesIgnoringExceptions(Directory dir, String... names) {
+ try {
+ dir.deleteFiles(Arrays.asList(names));
+ } catch (Throwable ignored) {
+ // ignore
}
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java Mon Oct 12 20:24:08 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
@@ -42,15 +43,7 @@ public class TestCodecHoldsOpenFiles ext
w.commit();
w.close();
- for(String fileName : d.listAll()) {
- try {
- d.deleteFile(fileName);
- // may succeed, e.g. if the file is completely read into RAM.
- } catch (IOException ioe) {
- // ignore: this means codec (correctly) is holding
- // the file open
- }
- }
+ d.deleteFiles(Arrays.asList(d.listAll()));
for(LeafReaderContext cxt : r.leaves()) {
TestUtil.checkReader(cxt.reader());
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java Mon Oct 12 20:24:08 2015
@@ -17,6 +17,15 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -31,14 +40,6 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
/*
Verify we can read the pre-2.1 file format, do searches
against it, and add documents to it.
@@ -299,7 +300,7 @@ public class TestDeletionPolicy extends
break;
}
- dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
gen--;
}
@@ -381,7 +382,7 @@ public class TestDeletionPolicy extends
while(gen > 0) {
IndexReader reader = DirectoryReader.open(dir);
reader.close();
- dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
gen--;
if (gen > 0) {
@@ -612,7 +613,7 @@ public class TestDeletionPolicy extends
}
}
if (i < N) {
- dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
}
gen--;
}
@@ -730,7 +731,7 @@ public class TestDeletionPolicy extends
}
}
if (i < N) {
- dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ dir.deleteFiles(Collections.singleton(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)));
}
gen--;
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java Mon Oct 12 20:24:08 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -39,8 +40,8 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -713,7 +714,7 @@ public class TestDirectoryReaderReopen e
// Blow away the index:
for(String fileName : dir.listAll()) {
- dir.deleteFile(fileName);
+ dir.deleteFiles(Collections.singleton(fileName));
}
w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
@@ -762,9 +763,7 @@ public class TestDirectoryReaderReopen e
DirectoryReader r = DirectoryReader.open(dir);
// Blow away the index:
- for(String fileName : dir.listAll()) {
- dir.deleteFile(fileName);
- }
+ dir.deleteFiles(Arrays.asList(dir.listAll()));
w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
doc = new Document();
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Mon Oct 12 20:24:08 2015
@@ -237,9 +237,7 @@ public class TestDoc extends LuceneTestC
Collection<String> filesToDelete = si.files();
codec.compoundFormat().write(dir, si, context);
si.setUseCompoundFile(true);
- for (final String fileToDelete : filesToDelete) {
- si1.info.dir.deleteFile(fileToDelete);
- }
+ si1.info.dir.deleteFiles(filesToDelete);
}
return new SegmentCommitInfo(si, 0, -1L, -1L, -1L);
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Mon Oct 12 20:24:08 2015
@@ -915,7 +915,7 @@ public class TestIndexWriterExceptions e
if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && stage.equals(trace[i].getMethodName())) {
isCommit = true;
}
- if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFile".equals(trace[i].getMethodName())) {
+ if (MockDirectoryWrapper.class.getName().equals(trace[i].getClassName()) && "deleteFiles".equals(trace[i].getMethodName())) {
isDelete = true;
}
if (SegmentInfos.class.getName().equals(trace[i].getClassName()) && "writeGlobalFieldMap".equals(trace[i].getMethodName())) {
@@ -964,7 +964,7 @@ public class TestIndexWriterExceptions e
} catch (RuntimeException re) {
// Expected
}
- assertTrue(failure.failOnCommit && failure.failOnDeleteFile);
+ assertTrue("failOnCommit=" + failure.failOnCommit + " failOnDeleteFile=" + failure.failOnDeleteFile, failure.failOnCommit && failure.failOnDeleteFile);
w.rollback();
String files[] = dir.listAll();
assertTrue(files.length == fileCount || (files.length == fileCount+1 && Arrays.asList(files).contains(IndexWriter.WRITE_LOCK_NAME)));
@@ -1209,7 +1209,7 @@ public class TestIndexWriterExceptions e
}
in.close();
out.close();
- dir.deleteFile(fileNameIn);
+ dir.deleteFiles(Collections.singleton(fileNameIn));
IndexReader reader = null;
try {
@@ -1263,7 +1263,7 @@ public class TestIndexWriterExceptions e
assertTrue(si.info.getUseCompoundFile());
List<String> victims = new ArrayList<String>(si.info.files());
Collections.shuffle(victims, random());
- dir.deleteFile(victims.get(0));
+ dir.deleteFiles(Collections.singleton(victims.get(0)));
corrupted = true;
break;
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/index/TestNRTReaderCleanup.java Mon Oct 12 20:24:08 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -25,8 +26,8 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
+import org.apache.lucene.util.LuceneTestCase;
/** LUCENE-5574 */
@SuppressFileSystems("WindowsFS") // the bug doesn't happen on windows.
@@ -66,9 +67,7 @@ public class TestNRTReaderCleanup extend
w.close();
// Blow away index and make a new writer:
- for(String fileName : dir.listAll()) {
- dir.deleteFile(fileName);
- }
+ dir.deleteFiles(Arrays.asList(dir.listAll()));
w = new RandomIndexWriter(random(), dir);
w.addDocument(doc);
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java Mon Oct 12 20:24:08 2015
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.util.IOUtils;
@@ -79,7 +80,7 @@ public class TestDirectory extends Lucen
}
// delete with a different dir
- dirs[(i+1)%dirs.length].deleteFile(fname);
+ dirs[(i+1)%dirs.length].deleteFiles(Collections.singleton(fname));
for (int j=0; j<dirs.length; j++) {
FSDirectory d2 = dirs[j];
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestNativeFSLockFactory.java Mon Oct 12 20:24:08 2015
@@ -20,6 +20,7 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Collections;
import org.apache.lucene.util.IOUtils;
@@ -85,7 +86,7 @@ public class TestNativeFSLockFactory ext
lock.ensureValid();
try {
- dir.deleteFile("test.lock");
+ dir.deleteFiles(Collections.singleton("test.lock"));
} catch (Exception e) {
// we can't delete a file for some reason, just clean up and assume the test.
IOUtils.closeWhileHandlingException(lock);
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestSimpleFSLockFactory.java Mon Oct 12 20:24:08 2015
@@ -19,6 +19,7 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.Collections;
import org.apache.lucene.util.IOUtils;
@@ -38,7 +39,7 @@ public class TestSimpleFSLockFactory ext
lock.ensureValid();
try {
- dir.deleteFile("test.lock");
+ dir.deleteFiles(Collections.singleton("test.lock"));
} catch (Exception e) {
// we can't delete a file for some reason, just clean up and assume the test.
IOUtils.closeWhileHandlingException(lock);
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/store/TestTrackingDirectoryWrapper.java Mon Oct 12 20:24:08 2015
@@ -43,7 +43,7 @@ public class TestTrackingDirectoryWrappe
TrackingDirectoryWrapper dir = new TrackingDirectoryWrapper(new RAMDirectory());
dir.createOutput("foo", newIOContext(random())).close();
assertEquals(asSet("foo"), dir.getCreatedFiles());
- dir.deleteFile("foo");
+ dir.deleteFiles(Collections.singleton("foo"));
assertEquals(Collections.emptySet(), dir.getCreatedFiles());
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java Mon Oct 12 20:24:08 2015
@@ -18,6 +18,7 @@ package org.apache.lucene.util.fst;
*/
import java.util.Arrays;
+import java.util.Collections;
import java.util.Random;
import org.apache.lucene.store.Directory;
@@ -31,6 +32,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.TimeUnits;
import org.apache.lucene.util.packed.PackedInts;
import org.junit.Ignore;
+
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
@Ignore("Requires tons of heap to run (30 GB hits OOME but 35 GB passes after ~4.5 hours)")
@@ -126,7 +128,7 @@ public class Test2BFST extends LuceneTes
fst = new FST<>(in, outputs);
in.close();
} else {
- dir.deleteFile("fst");
+ dir.deleteFiles(Collections.singleton("fst"));
}
}
}
@@ -203,7 +205,7 @@ public class Test2BFST extends LuceneTes
fst = new FST<>(in, outputs);
in.close();
} else {
- dir.deleteFile("fst");
+ dir.deleteFiles(Collections.singleton("fst"));
}
}
}
@@ -287,7 +289,7 @@ public class Test2BFST extends LuceneTes
fst = new FST<>(in, outputs);
in.close();
} else {
- dir.deleteFile("fst");
+ dir.deleteFiles(Collections.singleton("fst"));
}
}
}
Modified: lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/lucene6835/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Mon Oct 12 20:24:08 2015
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import java.nio.LongBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Random;
@@ -840,7 +841,7 @@ public class TestPackedInts extends Luce
assertEquals(mutable.get(i), reader.get(i));
}
in.close();
- directory.deleteFile("packed-ints.bin");
+ directory.deleteFiles(Collections.singleton("packed-ints.bin"));
}
directory.close();
}
Modified: lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java Mon Oct 12 20:24:08 2015
@@ -252,7 +252,7 @@ public abstract class BaseCompoundFormat
si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
try {
- cfs.deleteFile(testfile);
+ cfs.deleteFiles(Collections.singleton(testfile));
fail("didn't get expected exception");
} catch (UnsupportedOperationException expected) {
// expected UOE
Modified: lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java (original)
+++ lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java Mon Oct 12 20:24:08 2015
@@ -161,7 +161,7 @@ public abstract class BaseDirectoryTestC
int count = dir.listAll().length;
dir.createOutput("foo.txt", IOContext.DEFAULT).close();
assertEquals(count+1, dir.listAll().length);
- dir.deleteFile("foo.txt");
+ dir.deleteFiles(Collections.singleton("foo.txt"));
assertEquals(count, dir.listAll().length);
dir.close();
}
@@ -747,9 +747,7 @@ public abstract class BaseDirectoryTestC
}
in2.close();
- dir.deleteFile("test");
- dir.deleteFile("test2");
-
+ dir.deleteFiles(Arrays.asList(new String[] {"test", "test2"}));
dir.close();
}
Modified: lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Mon Oct 12 20:24:08 2015
@@ -302,7 +302,7 @@ public class MockDirectoryWrapper extend
if (damage == 0) {
action = "deleted";
- deleteFile(name, true);
+ deleteFiles(Collections.singleton(name), true);
} else if (damage == 1) {
action = "zeroed";
// Zero out file entirely
@@ -336,21 +336,21 @@ public class MockDirectoryWrapper extend
ii.close();
// Delete original and copy bytes back:
- deleteFile(name, true);
+ deleteFiles(Collections.singleton(name), true);
final IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
ii = in.openInput(tempFileName, LuceneTestCase.newIOContext(randomState));
out.copyBytes(ii, ii.length());
out.close();
ii.close();
- deleteFile(tempFileName, true);
+ deleteFiles(Collections.singleton(tempFileName), true);
} else if (damage == 3) {
// The file survived intact:
action = "didn't change";
} else {
action = "fully truncated";
// Totally truncate the file to zero bytes
- deleteFile(name, true);
+ deleteFiles(Collections.singleton(name), true);
IndexOutput out = in.createOutput(name, LuceneTestCase.newIOContext(randomState));
out.close();
}
@@ -466,9 +466,9 @@ public class MockDirectoryWrapper extend
}
@Override
- public synchronized void deleteFile(String name) throws IOException {
+ public synchronized void deleteFiles(Collection<String> names) throws IOException {
maybeYield();
- deleteFile(name, false);
+ deleteFiles(names, false);
}
// sets the cause of the incoming ioe to be the stack
@@ -492,7 +492,7 @@ public class MockDirectoryWrapper extend
}
}
- private synchronized void deleteFile(String name, boolean forced) throws IOException {
+ private synchronized void deleteFiles(Collection<String> names, boolean forced) throws IOException {
maybeYield();
maybeThrowDeterministicException();
@@ -500,6 +500,18 @@ public class MockDirectoryWrapper extend
if (crashed && !forced)
throw new IOException("cannot delete after crash");
+ for(String name : names) {
+ if (unSyncedFiles.contains(name)) {
+ unSyncedFiles.remove(name);
+ }
+ triedToDelete.remove(name);
+ }
+
+ in.deleteFiles(names);
+
+ // nocommit how to get virus checker back!!! must move down to MockFS?
+
+ /*
if (unSyncedFiles.contains(name))
unSyncedFiles.remove(name);
if (!forced && (noDeleteOpenFile || assertNoDeleteOpenFile)) {
@@ -524,9 +536,10 @@ public class MockDirectoryWrapper extend
}
triedToDelete.remove(name);
in.deleteFile(name);
+ */
}
- /** Returns true if {@link #deleteFile} was called with this
+ /** Returns true if {@link #deleteFiles} was called with this
* fileName, but the virus checker prevented the deletion. */
public boolean didTryToDelete(String fileName) {
return triedToDelete.contains(fileName);
Modified: lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java?rev=1708230&r1=1708229&r2=1708230&view=diff
==============================================================================
--- lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java (original)
+++ lucene/dev/branches/lucene6835/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java Mon Oct 12 20:24:08 2015
@@ -317,7 +317,7 @@ public class FSTTester<T> {
fst = new FST<>(in, outputs);
} finally {
in.close();
- dir.deleteFile("fst.bin");
+ dir.deleteFiles(Collections.singleton("fst.bin"));
}
}