You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2022/12/26 09:09:21 UTC
[lucene] branch branch_9x updated: Patch class files for Java 19 code to no longer have the "preview" flag (this enables Java 19 memory segments by default) (#12033)
This is an automated email from the ASF dual-hosted git repository.
uschindler pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new edc4ff6e64a Patch class files for Java 19 code to no longer have the "preview" flag (this enables Java 19 memory segments by default) (#12033)
edc4ff6e64a is described below
commit edc4ff6e64aa889a410151bbfa43657464c65c22
Author: Uwe Schindler <us...@apache.org>
AuthorDate: Mon Dec 26 10:07:44 2022 +0100
Patch class files for Java 19 code to no longer have the "preview" flag (this enables Java 19 memory segments by default) (#12033)
---
build.gradle | 1 +
gradle/generation/local-settings.gradle | 2 +-
gradle/java/javac.gradle | 45 -----------
gradle/java/memorysegment-mrjar.gradle | 89 ++++++++++++++++++++++
gradle/testing/defaults-tests.gradle | 4 -
lucene/CHANGES.txt | 5 ++
.../org/apache/lucene/store/MMapDirectory.java | 64 +++++++---------
.../org/apache/lucene/store/TestMmapDirectory.java | 2 -
8 files changed, 124 insertions(+), 88 deletions(-)
diff --git a/build.gradle b/build.gradle
index e6251156e8b..b3c1890de37 100644
--- a/build.gradle
+++ b/build.gradle
@@ -119,6 +119,7 @@ apply from: file('gradle/ide/eclipse.gradle')
// (java, tests)
apply from: file('gradle/java/folder-layout.gradle')
apply from: file('gradle/java/javac.gradle')
+apply from: file('gradle/java/memorysegment-mrjar.gradle')
apply from: file('gradle/testing/defaults-tests.gradle')
apply from: file('gradle/testing/randomization.gradle')
apply from: file('gradle/testing/fail-on-no-tests.gradle')
diff --git a/gradle/generation/local-settings.gradle b/gradle/generation/local-settings.gradle
index c7314681c54..3d0572167e5 100644
--- a/gradle/generation/local-settings.gradle
+++ b/gradle/generation/local-settings.gradle
@@ -96,7 +96,7 @@ tests.jvms=${testsJvms}
org.gradle.java.installations.auto-download=true
# Set these to enable automatic JVM location discovery.
-org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME
+org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME
#org.gradle.java.installations.paths=(custom paths)
""", "UTF-8")
diff --git a/gradle/java/javac.gradle b/gradle/java/javac.gradle
index a3344f07193..25f66c39de0 100644
--- a/gradle/java/javac.gradle
+++ b/gradle/java/javac.gradle
@@ -85,48 +85,3 @@ allprojects {
}
}
}
-
-configure(project(":lucene:core")) {
- plugins.withType(JavaPlugin) {
- sourceSets {
- main19 {
- java {
- srcDirs = ['src/java19']
- }
- }
- }
-
- configurations {
- // Inherit any dependencies from the main source set.
- main19Implementation.extendsFrom implementation
- }
-
- dependencies {
- // We need the main classes to compile our Java 19 pieces.
- main19Implementation sourceSets.main.output
- }
-
- tasks.named('compileMain19Java').configure {
- javaCompiler = javaToolchains.compilerFor {
- languageVersion = JavaLanguageVersion.of(19)
- }
-
- // undo alternative JDK support:
- options.forkOptions.javaHome = null
-
- sourceCompatibility = 19
- targetCompatibility = 19
- options.compilerArgs += ["--release", 19 as String, "--enable-preview"]
- }
-
- tasks.named('jar').configure {
- into('META-INF/versions/19') {
- from sourceSets.main19.output
- }
-
- manifest.attributes(
- 'Multi-Release': 'true'
- )
- }
- }
-}
diff --git a/gradle/java/memorysegment-mrjar.gradle b/gradle/java/memorysegment-mrjar.gradle
new file mode 100644
index 00000000000..f36f8873b39
--- /dev/null
+++ b/gradle/java/memorysegment-mrjar.gradle
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Produce an MR-JAR with Java 19 MemorySegment implementation for MMapDirectory
+
+configure(project(":lucene:core")) {
+ plugins.withType(JavaPlugin) {
+ sourceSets {
+ main19 {
+ java {
+ srcDirs = ['src/java19']
+ }
+ }
+ }
+
+ configurations {
+ // Inherit any dependencies from the main source set.
+ main19Implementation.extendsFrom implementation
+ }
+
+ dependencies {
+ // We need the main classes to compile our Java 19 pieces.
+ main19Implementation sourceSets.main.output
+ }
+
+ def patchClassFiles = { DirectoryProperty destinationDirectory, int expectedMajor ->
+ destinationDirectory.getAsFileTree().matching(new PatternSet().include('**/*.class')).visit{ details ->
+ if (!details.directory) {
+ logger.info("Patching: ${details.file}")
+ new RandomAccessFile(details.file, 'rw').withCloseable { f ->
+ int magic = f.readInt();
+ if (magic != (int)0xCAFEBABE) {
+ throw new GradleException("Invalid Java class file magic ${String.format("0x%08X", magic)}: ${details.file}")
+ }
+ f.seek(6L)
+ short major = f.readShort()
+ if (major != expectedMajor) {
+ throw new GradleException("Invalid Java class file version ${major}: ${details.file}")
+ }
+ // patch the minor version to 0 (remove preview flag):
+ f.seek(4L)
+ f.writeShort(0)
+ }
+ }
+ }
+ }
+
+ tasks.named('compileMain19Java').configure {
+ javaCompiler = javaToolchains.compilerFor {
+ languageVersion = JavaLanguageVersion.of(19)
+ }
+
+ // undo alternative JDK support:
+ options.forkOptions.javaHome = null
+
+ sourceCompatibility = 19
+ targetCompatibility = 19
+ options.compilerArgs += ["--release", 19 as String, "--enable-preview"]
+
+ doLast {
+ patchClassFiles(destinationDirectory, 63)
+ }
+ }
+
+ tasks.named('jar').configure {
+ into('META-INF/versions/19') {
+ from sourceSets.main19.output
+ }
+
+ manifest.attributes(
+ 'Multi-Release': 'true'
+ )
+ }
+ }
+}
diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle
index 6324800e213..d23fcdc267a 100644
--- a/gradle/testing/defaults-tests.gradle
+++ b/gradle/testing/defaults-tests.gradle
@@ -124,10 +124,6 @@ allprojects {
// (if the runner JVM does not support them, it will fail tests):
jvmArgs '--add-modules', 'jdk.unsupported,jdk.management'
- if (rootProject.runtimeJavaVersion == JavaVersion.VERSION_19) {
- jvmArgs '--enable-preview'
- }
-
systemProperty 'java.util.logging.config.file', file("${resources}/logging.properties")
systemProperty 'java.awt.headless', 'true'
systemProperty 'jdk.map.althashing.threshold', '0'
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 871f6d0ec54..36b7e7492cb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -68,6 +68,11 @@ New Features
use numeric fields that perform well both for filtering and sorting.
(Francisco Fernández Castaño)
+* GITHUB#12033: Support for Java 19 foreign memory support is now enabled by default,
+ no need to pass "--enable-preview" on the command line. If exactly Java 19 is used,
+ MMapDirectory will mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) and
+ indexes closed while queries are running can no longer crash the JVM. (Uwe Schindler)
+
Improvements
---------------------
* GITHUB#11785: Improve Tessellator performance by delaying calls to the method
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index 28abf2196df..c50a6dd1a46 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -61,8 +61,8 @@ import org.apache.lucene.util.Constants;
* the workaround will be automatically enabled (with no guarantees; if you discover any problems,
* you can disable it).
*
- * <p>On <b>Java 19</b> with {@code --enable-preview} command line setting, this class will use the
- * modern {@code MemorySegment} API which allows to safely unmap.
+ * <p>On exactly <b>Java 19</b> this class will use the modern {@code MemorySegment} API which
+ * allows to safely unmap.
*
* <p><b>NOTE:</b> Accessing this class either directly or indirectly from a thread while it's
* interrupted can close the underlying channel immediately if at the same time the thread is
@@ -106,8 +106,7 @@ public class MMapDirectory extends FSDirectory {
* Default max chunk size:
*
* <ul>
- * <li>16 GiBytes for 64 bit <b>Java 19</b> JVMs running with {@code --enable-preview} as
- * command line parameter
+ * <li>16 GiBytes for 64 bit <b>Java 19</b> JVMs
* <li>1 GiBytes for other 64 bit JVMs
* <li>256 MiBytes for 32 bit JVMs
* </ul>
@@ -211,9 +210,8 @@ public class MMapDirectory extends FSDirectory {
* non-Oracle/OpenJDK JVMs. It forcefully unmaps the buffer on close by using an undocumented
* internal cleanup functionality.
*
- * <p>On Java 19 with {@code --enable-preview} command line setting, this class will use the
- * modern {@code MemorySegment} API which allows to safely unmap. <em>The following warnings no
- * longer apply in that case!</em>
+ * <p>On exactly Java 19 this class will use the modern {@code MemorySegment} API which allows to
+ * safely unmap. <em>The following warnings no longer apply in that case!</em>
*
* <p><b>NOTE:</b> Enabling this is completely unsupported by Java and may lead to JVM crashes if
* <code>IndexInput</code> is closed while another thread is still accessing it (SIGSEGV).
@@ -373,39 +371,33 @@ public class MMapDirectory extends FSDirectory {
private static MMapIndexInputProvider lookupProvider() {
final var lookup = MethodHandles.lookup();
- try {
- final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
- // we use method handles, so we do not need to deal with setAccessible as we have private
- // access through the lookup:
- final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
+ final int runtimeVersion = Runtime.version().feature();
+ if (runtimeVersion == 19) {
try {
- return (MMapIndexInputProvider) constr.invoke();
- } catch (RuntimeException | Error e) {
- throw e;
- } catch (Throwable th) {
- throw new AssertionError(th);
+ final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
+ // we use method handles, so we do not need to deal with setAccessible as we have private
+ // access through the lookup:
+ final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class));
+ try {
+ return (MMapIndexInputProvider) constr.invoke();
+ } catch (RuntimeException | Error e) {
+ throw e;
+ } catch (Throwable th) {
+ throw new AssertionError(th);
+ }
+ } catch (NoSuchMethodException | IllegalAccessException e) {
+ throw new LinkageError(
+ "MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
+ } catch (ClassNotFoundException cnfe) {
+ throw new LinkageError(
+ "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
}
- } catch (
- @SuppressWarnings("unused")
- ClassNotFoundException e) {
- // we're before Java 19
- return new MappedByteBufferIndexInputProvider();
- } catch (
- @SuppressWarnings("unused")
- UnsupportedClassVersionError e) {
+ } else if (runtimeVersion >= 20) {
var log = Logger.getLogger(lookup.lookupClass().getName());
- if (Runtime.version().feature() == 19) {
- log.warning(
- "You are running with Java 19. To make full use of MMapDirectory, please pass '--enable-preview' to the Java command line.");
- } else {
- log.warning(
- "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene.");
- }
- return new MappedByteBufferIndexInputProvider();
- } catch (NoSuchMethodException | IllegalAccessException e) {
- throw new LinkageError(
- "MemorySegmentIndexInputProvider is missing correctly typed constructor", e);
+ log.warning(
+ "You are running with Java 20 or later. To make full use of MMapDirectory, please update Apache Lucene.");
}
+ return new MappedByteBufferIndexInputProvider();
}
static {
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
index 579b0c27108..5faf12484cc 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
@@ -52,8 +52,6 @@ public class TestMmapDirectory extends BaseDirectoryTestCase {
assertTrue(
"on Java 19 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
isMemorySegmentImpl());
- } else if (runtimeVersion > 19) {
- // TODO: We don't know how this is handled in later Java versions, so make no assumptions!
} else {
assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass());
}