You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/11/26 15:22:23 UTC
svn commit: r1641821 - in /lucene/dev/trunk/lucene:
core/src/test/org/apache/lucene/mockfile/
test-framework/src/java/org/apache/lucene/mockfile/
test-framework/src/java/org/apache/lucene/util/
Author: rmuir
Date: Wed Nov 26 14:22:23 2014
New Revision: 1641821
URL: http://svn.apache.org/r1641821
Log:
LUCENE-6072: add a way to test for too many open files
Added:
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleLimitFS.java (with props)
Modified:
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java?rev=1641821&r1=1641820&r2=1641821&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/mockfile/TestMockFilesystems.java Wed Nov 26 14:22:23 2014
@@ -17,6 +17,7 @@ package org.apache.lucene.mockfile;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -28,9 +29,12 @@ import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
@@ -206,4 +210,29 @@ public class TestMockFilesystems extends
assertTrue(seenMessage.get());
file.close();
}
+
+ public void testTooManyOpenFiles() throws IOException {
+ int n = 60;
+
+ Path dir = FilterPath.unwrap(createTempDir());
+ FileSystem fs = new HandleLimitFS(dir.getFileSystem(), n).getFileSystem(URI.create("file:///"));
+ dir = new FilterPath(dir, fs);
+
+ // create open files to exact limit
+ List<Closeable> toClose = new ArrayList<>();
+ for (int i = 0; i < n; i++) {
+ Path p = Files.createTempFile(dir, null, null);
+ toClose.add(Files.newOutputStream(p));
+ }
+
+ // now exceed
+ try {
+ Files.newOutputStream(Files.createTempFile(dir, null, null));
+ fail("didn't hit exception");
+ } catch (IOException e) {
+ assertTrue(e.getMessage().contains("Too many open files"));
+ }
+
+ IOUtils.close(toClose);
+ }
}
Added: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleLimitFS.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleLimitFS.java?rev=1641821&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleLimitFS.java (added)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleLimitFS.java Wed Nov 26 14:22:23 2014
@@ -0,0 +1,57 @@
+package org.apache.lucene.mockfile;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystemException;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * FileSystem that throws exception if file handles
+ * in use exceeds a specified limit
+ */
+public class HandleLimitFS extends HandleTrackingFS {
+ final int limit;
+ final AtomicInteger count = new AtomicInteger();
+
+ /**
+ * Create a new instance, limiting the maximum number
+ * of open files to {@code limit}
+ * @param delegate delegate filesystem to wrap.
+ * @param limit maximum number of open files.
+ */
+ public HandleLimitFS(FileSystem delegate, int limit) {
+ super("handlelimit://", delegate);
+ this.limit = limit;
+ }
+
+ @Override
+ protected void onOpen(Path path, Object stream) throws IOException {
+ if (count.incrementAndGet() > limit) {
+ count.decrementAndGet();
+ throw new FileSystemException(path.toString(), null, "Too many open files");
+ }
+ }
+
+ @Override
+ protected void onClose(Path path, Object stream) throws IOException {
+ count.decrementAndGet();
+ }
+}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java?rev=1641821&r1=1641820&r2=1641821&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/mockfile/HandleTrackingFS.java Wed Nov 26 14:22:23 2014
@@ -17,6 +17,7 @@ package org.apache.lucene.mockfile;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -34,6 +35,8 @@ import java.nio.file.attribute.FileAttri
import java.util.Set;
import java.util.concurrent.ExecutorService;
+import org.apache.lucene.util.IOUtils;
+
/**
* Base class for tracking file handles.
* <p>
@@ -63,7 +66,6 @@ public abstract class HandleTrackingFS e
* @throws IOException if an I/O error occurs.
*/
protected abstract void onOpen(Path path, Object stream) throws IOException;
-
/**
* Called when {@code path} is closed via {@code stream}.
@@ -73,6 +75,21 @@ public abstract class HandleTrackingFS e
*/
protected abstract void onClose(Path path, Object stream) throws IOException;
+ /**
+ * Helper method, to deal with onOpen() throwing exception
+ */
+ final void callOpenHook(Path path, Closeable stream) throws IOException {
+ boolean success = false;
+ try {
+ onOpen(path, stream);
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(stream);
+ }
+ }
+ }
+
@Override
public InputStream newInputStream(Path path, OpenOption... options) throws IOException {
InputStream stream = new FilterInputStream2(super.newInputStream(path, options)) {
@@ -97,7 +114,7 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, stream);
+ callOpenHook(path, stream);
return stream;
}
@@ -125,7 +142,7 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, stream);
+ callOpenHook(path, stream);
return stream;
}
@@ -153,7 +170,7 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, channel);
+ callOpenHook(path, channel);
return channel;
}
@@ -181,7 +198,7 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, channel);
+ callOpenHook(path, channel);
return channel;
}
@@ -209,7 +226,7 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, channel);
+ callOpenHook(path, channel);
return channel;
}
@@ -242,7 +259,7 @@ public abstract class HandleTrackingFS e
}
};
}
- onOpen(dir, stream);
+ callOpenHook(dir, stream);
return stream;
}
@@ -279,7 +296,7 @@ public abstract class HandleTrackingFS e
@Override
public SecureDirectoryStream<Path> newDirectoryStream(Path path, LinkOption... options) throws IOException {
SecureDirectoryStream<Path> stream = new TrackingSecureDirectoryStream(super.newDirectoryStream(path, options), path);
- onOpen(path, stream);
+ callOpenHook(path, stream);
return stream;
}
@@ -307,7 +324,7 @@ public abstract class HandleTrackingFS e
return this == obj;
}
};
- onOpen(path, channel);
+ callOpenHook(path, channel);
return channel;
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java?rev=1641821&r1=1641820&r2=1641821&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java Wed Nov 26 14:22:23 2014
@@ -13,6 +13,7 @@ import java.util.Locale;
import java.util.Random;
import org.apache.lucene.mockfile.DisableFsyncFS;
+import org.apache.lucene.mockfile.HandleLimitFS;
import org.apache.lucene.mockfile.LeakFS;
import org.apache.lucene.mockfile.VerboseFS;
import org.apache.lucene.mockfile.WindowsFS;
@@ -106,6 +107,10 @@ final class TestRuleTemporaryFilesCleanu
javaTempDir = initializeJavaTempDir();
}
+ // os/config-independent limit for too many open files
+ // TODO: can we make this lower?
+ private static final int MAX_OPEN_FILES = 2048;
+
private FileSystem initializeFileSystem() {
FileSystem fs = FileSystems.getDefault();
if (LuceneTestCase.VERBOSE) {
@@ -116,6 +121,7 @@ final class TestRuleTemporaryFilesCleanu
if (random.nextInt(10) > 0) {
fs = new DisableFsyncFS(fs).getFileSystem(null);
fs = new LeakFS(fs).getFileSystem(null);
+ fs = new HandleLimitFS(fs, MAX_OPEN_FILES).getFileSystem(null);
// windows is currently slow
if (random.nextInt(10) == 0) {
fs = new WindowsFS(fs).getFileSystem(null);