You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2016/02/15 19:58:02 UTC
lucene-solr git commit: LUCENE-6989: Add preliminary support for
MMapDirectory unmapping in Java 9
Repository: lucene-solr
Updated Branches:
refs/heads/master a9dc40294 -> 7b6df2542
LUCENE-6989: Add preliminary support for MMapDirectory unmapping in Java 9
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7b6df254
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7b6df254
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7b6df254
Branch: refs/heads/master
Commit: 7b6df2542d345ef1815693d43c9e18c6e64726bd
Parents: a9dc402
Author: Uwe Schindler <us...@apache.org>
Authored: Mon Feb 15 19:57:39 2016 +0100
Committer: Uwe Schindler <us...@apache.org>
Committed: Mon Feb 15 19:57:39 2016 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/index/IndexWriter.java | 6 +-
.../org/apache/lucene/store/MMapDirectory.java | 124 +++++++++++++------
.../apache/lucene/store/TestMmapDirectory.java | 3 +-
.../org/apache/lucene/store/TestMultiMMap.java | 3 +-
lucene/tools/junit4/tests.policy | 1 +
6 files changed, 100 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b6df254/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c3f305d..fc95bb2 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -61,6 +61,9 @@ New Features
* LUCENE-6975: Add ExactPointQuery, to match a single N-dimensional
point (Robert Muir, Mike McCandless)
+* LUCENE-6989: Add preliminary support for MMapDirectory unmapping in Java 9.
+ (Uwe Schindler, Chris Hegarty, Peter Levart)
+
API Changes
* LUCENE-6067: Accountable.getChildResources has a default
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b6df254/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 8886ab1..57ce3dd 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1055,7 +1055,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
"index=" + segString() + "\n" +
"version=" + Version.LATEST.toString() + "\n" +
config.toString());
- infoStream.message("IW", "MMapDirectory.UNMAP_SUPPORTED=" + MMapDirectory.UNMAP_SUPPORTED);
+ final StringBuilder unmapInfo = new StringBuilder(Boolean.toString(MMapDirectory.UNMAP_SUPPORTED));
+ if (!MMapDirectory.UNMAP_SUPPORTED) {
+ unmapInfo.append(" (").append(MMapDirectory.UNMAP_NOT_SUPPORTED_REASON).append(")");
+ }
+ infoStream.message("IW", "MMapDirectory.UNMAP_SUPPORTED=" + unmapInfo);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b6df254/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index dca843d..da8cf70 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -17,6 +17,9 @@
package org.apache.lucene.store;
+import static java.lang.invoke.MethodHandles.*;
+import static java.lang.invoke.MethodType.methodType;
+
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
@@ -27,10 +30,10 @@ import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.AccessController;
import java.security.PrivilegedAction;
-import java.security.PrivilegedExceptionAction;
-import java.security.PrivilegedActionException;
import java.util.Locale;
+import java.util.Objects;
import java.util.concurrent.Future;
+import java.lang.invoke.MethodHandle;
import java.lang.reflect.Method;
import org.apache.lucene.store.ByteBufferIndexInput.BufferCleaner;
@@ -161,24 +164,6 @@ public class MMapDirectory extends FSDirectory {
}
/**
- * <code>true</code>, if this platform supports unmapping mmapped files.
- */
- public static final boolean UNMAP_SUPPORTED =
- AccessController.doPrivileged((PrivilegedAction<Boolean>) MMapDirectory::checkUnmapSupported);
-
- @SuppressForbidden(reason = "Java 9 Jigsaw whitelists access to sun.misc.Cleaner, so setAccessible works")
- private static boolean checkUnmapSupported() {
- try {
- Class<?> clazz = Class.forName("java.nio.DirectByteBuffer");
- Method method = clazz.getMethod("cleaner");
- method.setAccessible(true);
- return true;
- } catch (Exception e) {
- return false;
- }
- }
-
- /**
* This method enables the workaround for unmapping the buffers
* from address space after closing {@link IndexInput}, that is
* mentioned in the bug report. This hack may fail on non-Sun JVMs.
@@ -191,8 +176,9 @@ public class MMapDirectory extends FSDirectory {
* is <code>false</code> and the workaround cannot be enabled.
*/
public void setUseUnmap(final boolean useUnmapHack) {
- if (useUnmapHack && !UNMAP_SUPPORTED)
- throw new IllegalArgumentException("Unmap hack not supported on this platform!");
+ if (useUnmapHack && !UNMAP_SUPPORTED) {
+ throw new IllegalArgumentException(UNMAP_NOT_SUPPORTED_REASON);
+ }
this.useUnmapHack=useUnmapHack;
}
@@ -310,23 +296,87 @@ public class MMapDirectory extends FSDirectory {
return newIoe;
}
- private static final BufferCleaner CLEANER = (final ByteBufferIndexInput parent, final ByteBuffer buffer) -> {
+ /**
+ * <code>true</code>, if this platform supports unmapping mmapped files.
+ */
+ public static final boolean UNMAP_SUPPORTED;
+
+ /**
+ * if {@link #UNMAP_SUPPORTED} is {@code false}, this contains the reason why unmapping is not supported.
+ */
+ public static final String UNMAP_NOT_SUPPORTED_REASON;
+
+ /** Reference to a BufferCleaner that does unmapping; {@code null} if not supported. */
+ private static final BufferCleaner CLEANER;
+
+ static {
+ final Object hack = AccessController.doPrivileged((PrivilegedAction<Object>) MMapDirectory::initUnmapHack);
+ if (hack instanceof BufferCleaner) {
+ CLEANER = (BufferCleaner) hack;
+ UNMAP_SUPPORTED = true;
+ UNMAP_NOT_SUPPORTED_REASON = null;
+ } else {
+ CLEANER = null;
+ UNMAP_SUPPORTED = false;
+ UNMAP_NOT_SUPPORTED_REASON = hack.toString();
+ }
+ }
+
+ @SuppressForbidden(reason = "Needs access to private APIs in DirectBuffer and sun.misc.Cleaner to enable hack")
+ private static Object initUnmapHack() {
+ final Lookup lookup = lookup();
try {
- AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
- @Override
- @SuppressForbidden(reason = "Java 9 Jigsaw whitelists access to sun.misc.Cleaner, so setAccessible works")
- public Void run() throws Exception {
- final Method getCleanerMethod = buffer.getClass().getMethod("cleaner");
- getCleanerMethod.setAccessible(true);
- final Object cleaner = getCleanerMethod.invoke(buffer);
- if (cleaner != null) {
- cleaner.getClass().getMethod("clean").invoke(cleaner);
+ final Class<?> directBufferClass = Class.forName("java.nio.DirectByteBuffer");
+
+ final Method m = directBufferClass.getMethod("cleaner");
+ m.setAccessible(true);
+ final MethodHandle directBufferCleanerMethod = lookup.unreflect(m);
+ final Class<?> cleanerClass = directBufferCleanerMethod.type().returnType();
+
+ final MethodHandle cleanMethod;
+ if (Runnable.class.isAssignableFrom(cleanerClass)) {
+ // early Java 9 impl using Runnable (we do the security check early that the Runnable does at runtime):
+ final SecurityManager sm = System.getSecurityManager();
+ if (sm != null) {
+ sm.checkPackageAccess("jdk.internal.ref");
+ }
+ cleanMethod = explicitCastArguments(lookup.findVirtual(Runnable.class, "run", methodType(void.class)),
+ methodType(void.class, cleanerClass));
+ } else {
+ // can be either the old internal "sun.misc.Cleaner" or
+ // the new Java 9 "java.lang.ref.Cleaner$Cleanable":
+ cleanMethod = lookup.findVirtual(cleanerClass, "clean", methodType(void.class));
+ }
+
+ final MethodHandle nonNullTest = explicitCastArguments(lookup.findStatic(Objects.class, "nonNull", methodType(boolean.class, Object.class)),
+ methodType(boolean.class, cleanerClass));
+ final MethodHandle noop = dropArguments(explicitCastArguments(constant(Void.class, null), methodType(void.class)), 0, cleanerClass);
+ final MethodHandle unmapper = explicitCastArguments(filterReturnValue(directBufferCleanerMethod, guardWithTest(nonNullTest, cleanMethod, noop)),
+ methodType(void.class, ByteBuffer.class));
+
+ return (BufferCleaner) (ByteBufferIndexInput parent, ByteBuffer buffer) -> {
+ if (directBufferClass.isInstance(buffer)) {
+ final Throwable error = AccessController.doPrivileged((PrivilegedAction<Throwable>) () -> {
+ try {
+ unmapper.invokeExact(buffer);
+ return null;
+ } catch (Throwable t) {
+ return t;
+ }
+ });
+ if (error != null) {
+ throw new IOException("Unable to unmap the mapped buffer: " + parent.toString(), error);
}
- return null;
}
- });
- } catch (PrivilegedActionException e) {
- throw new IOException("Unable to unmap the mapped buffer: " + parent.toString(), e.getCause());
+ };
+ } catch (ReflectiveOperationException e) {
+ return "Unmapping is not supported on this platform, because internal Java APIs are not compatible to this Lucene version: " + e;
+ } catch (SecurityException e) {
+ return "Unmapping is not supported, because not all required permissions are given to the Lucene JAR file. " +
+ "Please grant at least the following permissions: RuntimePermission(\"accessClassInPackage.sun.misc\"), " +
+ "RuntimePermission(\"accessClassInPackage.jdk.internal.ref\"), and " +
+ "ReflectPermission(\"suppressAccessChecks\")";
}
- };
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b6df254/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
index 44e6de2..153cc5e 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
@@ -35,7 +35,8 @@ public class TestMmapDirectory extends BaseDirectoryTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
- assumeTrue("test requires a jre that supports unmapping", MMapDirectory.UNMAP_SUPPORTED);
+ assumeTrue("test requires a jre that supports unmapping: " + MMapDirectory.UNMAP_NOT_SUPPORTED_REASON,
+ MMapDirectory.UNMAP_SUPPORTED);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b6df254/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java b/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
index adea8ff..3b6ef22 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
@@ -46,7 +46,8 @@ public class TestMultiMMap extends BaseDirectoryTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
- assumeTrue("test requires a jre that supports unmapping", MMapDirectory.UNMAP_SUPPORTED);
+ assumeTrue("test requires a jre that supports unmapping: " + MMapDirectory.UNMAP_NOT_SUPPORTED_REASON,
+ MMapDirectory.UNMAP_SUPPORTED);
}
public void testCloneSafety() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7b6df254/lucene/tools/junit4/tests.policy
----------------------------------------------------------------------
diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy
index a630513..a579fe2 100644
--- a/lucene/tools/junit4/tests.policy
+++ b/lucene/tools/junit4/tests.policy
@@ -64,6 +64,7 @@ grant {
permission java.lang.RuntimePermission "createClassLoader";
// needed to test unmap hack on platforms that support it
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+ permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.ref";
// needed by cyberneko usage by benchmarks on J9
permission java.lang.RuntimePermission "accessClassInPackage.org.apache.xerces.util";
// needed by jacoco to dump coverage