You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/11/26 16:31:57 UTC
svn commit: r1641833 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/test/org/apache/lucene/mockfile/
lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/mockfile/
lucene/test-framework/src/java/org/apac...
Author: rmuir
Date: Wed Nov 26 15:31:57 2014
New Revision: 1641833
URL: http://svn.apache.org/r1641833
Log:
LUCENE-6072: use mockfilesystem in tests
Added:
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/mockfile/
- copied from r1641632, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/mockfile/
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/
- copied from r1641632, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleLimitFS.java
- copied unchanged from r1641821, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleLimitFS.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/build.xml
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java?rev=1641833&r1=1641632&r2=1641833&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java Wed Nov 26 15:31:57 2014
@@ -17,6 +17,7 @@ package org.apache.lucene.mockfile;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -28,8 +29,12 @@ import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
@@ -123,6 +128,7 @@ public class TestMockFilesystems extends
}
public void testDeleteOpenFile() throws IOException {
+ assumeFalse("windows is not supported", Constants.WINDOWS);
Path dir = FilterPath.unwrap(createTempDir());
FileSystem fs = new WindowsFS(dir.getFileSystem()).getFileSystem(URI.create("file:///"));
Path wrapped = new FilterPath(dir, fs);
@@ -141,6 +147,7 @@ public class TestMockFilesystems extends
}
public void testDeleteIfExistsOpenFile() throws IOException {
+ assumeFalse("windows is not supported", Constants.WINDOWS);
Path dir = FilterPath.unwrap(createTempDir());
FileSystem fs = new WindowsFS(dir.getFileSystem()).getFileSystem(URI.create("file:///"));
Path wrapped = new FilterPath(dir, fs);
@@ -159,6 +166,7 @@ public class TestMockFilesystems extends
}
public void testRenameOpenFile() throws IOException {
+ assumeFalse("windows is not supported", Constants.WINDOWS);
Path dir = FilterPath.unwrap(createTempDir());
FileSystem fs = new WindowsFS(dir.getFileSystem()).getFileSystem(URI.create("file:///"));
Path wrapped = new FilterPath(dir, fs);
@@ -202,4 +210,29 @@ public class TestMockFilesystems extends
assertTrue(seenMessage.get());
file.close();
}
+
+ public void testTooManyOpenFiles() throws IOException {
+ int n = 60;
+
+ Path dir = FilterPath.unwrap(createTempDir());
+ FileSystem fs = new HandleLimitFS(dir.getFileSystem(), n).getFileSystem(URI.create("file:///"));
+ dir = new FilterPath(dir, fs);
+
+ // create open files to exact limit
+ List<Closeable> toClose = new ArrayList<>();
+ for (int i = 0; i < n; i++) {
+ Path p = Files.createTempFile(dir, null, null);
+ toClose.add(Files.newOutputStream(p));
+ }
+
+ // now exceed
+ try {
+ Files.newOutputStream(Files.createTempFile(dir, null, null));
+ fail("didn't hit exception");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("Too many open files"));
+ }
+
+ IOUtils.close(toClose);
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/build.xml?rev=1641833&r1=1641832&r2=1641833&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/build.xml (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/build.xml Wed Nov 26 15:31:57 2014
@@ -22,6 +22,11 @@
<property name="build.dir" location="../build/test-framework"/>
+ <!-- file is part of the API -->
+ <property name="forbidden-base-excludes" value="
+ org/apache/lucene/mockfile/FilterPath.class
+ "/>
+
<import file="../common-build.xml"/>
<path id="classpath">
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java?rev=1641833&r1=1641632&r2=1641833&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/FilterFileSystem.java Wed Nov 26 15:31:57 2014
@@ -97,38 +97,44 @@ public class FilterFileSystem extends Fi
@Override
public Iterable<Path> getRootDirectories() {
final Iterable<Path> roots = delegate.getRootDirectories();
- return () -> {
- final Iterator<Path> iterator = roots.iterator();
- return new Iterator<Path>() {
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public Path next() {
- return new FilterPath(iterator.next(), FilterFileSystem.this);
- }
- };
+ return new Iterable<Path>() {
+ @Override
+ public Iterator<Path> iterator() {
+ final Iterator<Path> iterator = roots.iterator();
+ return new Iterator<Path>() {
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public Path next() {
+ return new FilterPath(iterator.next(), FilterFileSystem.this);
+ }
+ };
+ }
};
}
@Override
public Iterable<FileStore> getFileStores() {
final Iterable<FileStore> fileStores = delegate.getFileStores();
- return () -> {
- final Iterator<FileStore> iterator = fileStores.iterator();
- return new Iterator<FileStore>() {
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public FileStore next() {
- return new FilterFileStore(iterator.next(), parent.getScheme());
- }
- };
+ return new Iterable<FileStore>() {
+ @Override
+ public Iterator<FileStore> iterator() {
+ final Iterator<FileStore> iterator = fileStores.iterator();
+ return new Iterator<FileStore>() {
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public FileStore next() {
+ return new FilterFileStore(iterator.next(), parent.getScheme());
+ }
+ };
+ }
};
}
@@ -145,11 +151,14 @@ public class FilterFileSystem extends Fi
@Override
public PathMatcher getPathMatcher(String syntaxAndPattern) {
final PathMatcher matcher = delegate.getPathMatcher(syntaxAndPattern);
- return path -> {
- if (path instanceof FilterPath) {
- return matcher.matches(((FilterPath)path).delegate);
+ return new PathMatcher() {
+ @Override
+ public boolean matches(Path path) {
+ if (path instanceof FilterPath) {
+ return matcher.matches(((FilterPath)path).delegate);
+ }
+ return false;
}
- return false;
};
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java?rev=1641833&r1=1641632&r2=1641833&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java Wed Nov 26 15:31:57 2014
@@ -17,6 +17,7 @@ package org.apache.lucene.mockfile;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -34,6 +35,8 @@ import java.nio.file.attribute.FileAttri
import java.util.Set;
import java.util.concurrent.ExecutorService;
+import org.apache.lucene.util.IOUtils;
+
/**
* Base class for tracking file handles.
* <p>
@@ -63,7 +66,6 @@ public abstract class HandleTrackingFS e
* @throws IOException if an I/O error occurs.
*/
protected abstract void onOpen(Path path, Object stream) throws IOException;
-
/**
* Called when {@code path} is closed via {@code stream}.
@@ -73,8 +75,23 @@ public abstract class HandleTrackingFS e
*/
protected abstract void onClose(Path path, Object stream) throws IOException;
+ /**
+ * Helper method, to deal with onOpen() throwing exception
+ */
+ final void callOpenHook(Path path, Closeable stream) throws IOException {
+ boolean success = false;
+ try {
+ onOpen(path, stream);
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(stream);
+ }
+ }
+ }
+
@Override
- public InputStream newInputStream(Path path, OpenOption... options) throws IOException {
+ public InputStream newInputStream(final Path path, OpenOption... options) throws IOException {
InputStream stream = new FilterInputStream2(super.newInputStream(path, options)) {
@Override
public void close() throws IOException {
@@ -97,7 +114,7 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, stream);
+ callOpenHook(path, stream);
return stream;
}
@@ -125,12 +142,12 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, stream);
+ callOpenHook(path, stream);
return stream;
}
@Override
- public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+ public FileChannel newFileChannel(final Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
FileChannel channel = new FilterFileChannel(super.newFileChannel(path, options, attrs)) {
@Override
protected void implCloseChannel() throws IOException {
@@ -153,12 +170,12 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, channel);
+ callOpenHook(path, channel);
return channel;
}
@Override
- public AsynchronousFileChannel newAsynchronousFileChannel(Path path, Set<? extends OpenOption> options, ExecutorService executor, FileAttribute<?>... attrs) throws IOException {
+ public AsynchronousFileChannel newAsynchronousFileChannel(final Path path, Set<? extends OpenOption> options, ExecutorService executor, FileAttribute<?>... attrs) throws IOException {
AsynchronousFileChannel channel = new FilterAsynchronousFileChannel(super.newAsynchronousFileChannel(path, options, executor, attrs)) {
@Override
public void close() throws IOException {
@@ -181,12 +198,12 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, channel);
+ callOpenHook(path, channel);
return channel;
}
@Override
- public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+ public SeekableByteChannel newByteChannel(final Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
SeekableByteChannel channel = new FilterSeekableByteChannel(super.newByteChannel(path, options, attrs)) {
@Override
public void close() throws IOException {
@@ -209,12 +226,12 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, channel);
+ callOpenHook(path, channel);
return channel;
}
@Override
- public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) throws IOException {
+ public DirectoryStream<Path> newDirectoryStream(final Path dir, Filter<? super Path> filter) throws IOException {
DirectoryStream<Path> stream = super.newDirectoryStream(dir, filter);
if (stream instanceof SecureDirectoryStream) {
stream = new TrackingSecureDirectoryStream((SecureDirectoryStream<Path>)stream, dir);
@@ -242,7 +259,7 @@ public abstract class HandleTrackingFS e
}
};
}
- onOpen(dir, stream);
+ callOpenHook(dir, stream);
return stream;
}
@@ -279,12 +296,12 @@ public abstract class HandleTrackingFS e
@Override
public SecureDirectoryStream<Path> newDirectoryStream(Path path, LinkOption... options) throws IOException {
SecureDirectoryStream<Path> stream = new TrackingSecureDirectoryStream(super.newDirectoryStream(path, options), path);
- onOpen(path, stream);
+ callOpenHook(path, stream);
return stream;
}
@Override
- public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+ public SeekableByteChannel newByteChannel(final Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
SeekableByteChannel channel = new FilterSeekableByteChannel(super.newByteChannel(path, options, attrs)) {
@Override
public void close() throws IOException {
@@ -307,7 +324,7 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, channel);
+ callOpenHook(path, channel);
return channel;
}
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java?rev=1641833&r1=1641832&r2=1641833&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java Wed Nov 26 15:31:57 2014
@@ -1,14 +1,22 @@
package org.apache.lucene.util;
import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
+import java.util.Random;
+import org.apache.lucene.mockfile.DisableFsyncFS;
+import org.apache.lucene.mockfile.HandleLimitFS;
+import org.apache.lucene.mockfile.LeakFS;
+import org.apache.lucene.mockfile.VerboseFS;
+import org.apache.lucene.mockfile.WindowsFS;
import org.apache.lucene.util.LuceneTestCase.SuppressTempFileChecks;
import com.carrotsearch.randomizedtesting.RandomizedContext;
@@ -52,6 +60,11 @@ final class TestRuleTemporaryFilesCleanu
* Per-test class temporary folder.
*/
private Path tempDirBase;
+
+ /**
+ * Per-test filesystem
+ */
+ private FileSystem fileSystem;
/**
* Suite failure marker.
@@ -90,11 +103,39 @@ final class TestRuleTemporaryFilesCleanu
super.before();
assert tempDirBase == null;
+ fileSystem = initializeFileSystem();
javaTempDir = initializeJavaTempDir();
}
+
+ // os/config-independent limit for too many open files
+ // TODO: can we make this lower?
+ private static final int MAX_OPEN_FILES = 2048;
+
+ private FileSystem initializeFileSystem() {
+ FileSystem fs = FileSystems.getDefault();
+ if (LuceneTestCase.VERBOSE) {
+ fs = new VerboseFS(fs, new TestRuleSetupAndRestoreClassEnv.ThreadNameFixingPrintStreamInfoStream(System.out)).getFileSystem(null);
+ }
+ Random random = RandomizedContext.current().getRandom();
+ // sometimes just use a bare filesystem
+ if (random.nextInt(10) > 0) {
+ fs = new DisableFsyncFS(fs).getFileSystem(null);
+ fs = new LeakFS(fs).getFileSystem(null);
+ fs = new HandleLimitFS(fs, MAX_OPEN_FILES).getFileSystem(null);
+ // windows is currently slow
+ if (random.nextInt(10) == 0) {
+ fs = new WindowsFS(fs).getFileSystem(null);
+ }
+ }
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("filesystem: " + fs.provider());
+ }
+ return fs.provider().getFileSystem(URI.create("file:///"));
+ }
private Path initializeJavaTempDir() throws IOException {
- Path javaTempDir = Paths.get(System.getProperty("tempDir", System.getProperty("java.io.tmpdir")));
+ Path javaTempDir = fileSystem.getPath(System.getProperty("tempDir", System.getProperty("java.io.tmpdir")));
+
Files.createDirectories(javaTempDir);
assert Files.isDirectory(javaTempDir) &&
@@ -134,6 +175,9 @@ final class TestRuleTemporaryFilesCleanu
}
throw e;
}
+ if (fileSystem != FileSystems.getDefault()) {
+ fileSystem.close();
+ }
} else {
if (tempDirBasePath != null) {
System.err.println("NOTE: leaving temporary files on disk at: " + tempDirBasePath);
@@ -171,7 +215,7 @@ final class TestRuleTemporaryFilesCleanu
}
return tempDirBase;
}
-
+
/**
* @see LuceneTestCase#createTempDir()
*/