You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2023/06/12 19:07:10 UTC
[lucene] branch main updated: Implement MMapDirectory with Java 21 Project Panama Preview API (#12294)
This is an automated email from the ASF dual-hosted git repository.
uschindler pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new c8e05c8cd6e Implement MMapDirectory with Java 21 Project Panama Preview API (#12294)
c8e05c8cd6e is described below
commit c8e05c8cd6e018133d3d739d159d5b3b6b29c179
Author: Uwe Schindler <us...@apache.org>
AuthorDate: Mon Jun 12 21:07:04 2023 +0200
Implement MMapDirectory with Java 21 Project Panama Preview API (#12294)
---
buildSrc/scriptDepVersions.gradle | 2 +-
gradle/generation/extract-jdk-apis.gradle | 2 +-
.../extract-jdk-apis/ExtractJdkApis.java | 3 +-
gradle/template.gradle.properties | 2 +-
lucene/CHANGES.txt | 6 +
lucene/core/src/generated/jdk/jdk21.apijar | Bin 0 -> 16543 bytes
.../org/apache/lucene/store/MMapDirectory.java | 14 +-
.../lucene/store/MemorySegmentIndexInput.java | 588 +++++++++++++++++++++
.../store/MemorySegmentIndexInputProvider.java | 120 +++++
.../org/apache/lucene/store/TestMmapDirectory.java | 4 +-
.../lucene/distribution/TestModularLayer.java | 2 +-
11 files changed, 729 insertions(+), 14 deletions(-)
diff --git a/buildSrc/scriptDepVersions.gradle b/buildSrc/scriptDepVersions.gradle
index 8751da63249..a6eae860b1c 100644
--- a/buildSrc/scriptDepVersions.gradle
+++ b/buildSrc/scriptDepVersions.gradle
@@ -22,7 +22,7 @@
ext {
scriptDepVersions = [
"apache-rat": "0.14",
- "asm": "9.4",
+ "asm": "9.5",
"commons-codec": "1.13",
"ecj": "3.30.0",
"flexmark": "0.61.24",
diff --git a/gradle/generation/extract-jdk-apis.gradle b/gradle/generation/extract-jdk-apis.gradle
index 834a490cc0d..6a5fb5ad8fd 100644
--- a/gradle/generation/extract-jdk-apis.gradle
+++ b/gradle/generation/extract-jdk-apis.gradle
@@ -27,7 +27,7 @@ configure(rootProject) {
configure(project(":lucene:core")) {
ext {
apijars = file('src/generated/jdk');
- mrjarJavaVersions = [ 19, 20 ]
+ mrjarJavaVersions = [ 19, 20, 21 ]
}
configurations {
diff --git a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
index 8d185f9b429..7121374850b 100644
--- a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
+++ b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java
@@ -55,7 +55,8 @@ public final class ExtractJdkApis {
static final Map<Integer,List<String>> CLASSFILE_PATTERNS = Map.of(
19, List.of(PATTERN_PANAMA_FOREIGN),
- 20, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR)
+ 20, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR),
+ 21, List.of(PATTERN_PANAMA_FOREIGN)
);
public static void main(String... args) throws IOException {
diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties
index a626d39f3bb..9ac8c42e9dd 100644
--- a/gradle/template.gradle.properties
+++ b/gradle/template.gradle.properties
@@ -102,5 +102,5 @@ tests.jvms=@TEST_JVMS@
org.gradle.java.installations.auto-download=true
# Set these to enable automatic JVM location discovery.
-org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME
+org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,JAVA22_HOME,RUNTIME_JAVA_HOME
#org.gradle.java.installations.paths=(custom paths)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 48626a8349c..1f478e126cc 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -147,6 +147,12 @@ New Features
thoroughly and report bugs/slowness to Lucene's mailing list.
(Chris Hegarty, Robert Muir, Uwe Schindler)
+* GITHUB#12294: Add support for Java 21 foreign memory API. If Java 19 up to 21 is used,
+ MMapDirectory will mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) and indexes
+ closed while queries are running can no longer crash the JVM. To disable this feature,
+ pass the following sysprop on Java command line:
+ "-Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false" (Uwe Schindler)
+
Improvements
---------------------
diff --git a/lucene/core/src/generated/jdk/jdk21.apijar b/lucene/core/src/generated/jdk/jdk21.apijar
new file mode 100644
index 00000000000..69906ccf4b0
Binary files /dev/null and b/lucene/core/src/generated/jdk/jdk21.apijar differ
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index 9ca636b4cd7..c9cddb4a308 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -76,9 +76,9 @@ import org.apache.lucene.util.SuppressForbidden;
* <li>{@code permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";}
* </ul>
*
- * <p>On exactly <b>Java 19</b> and <b>Java 20</b> this class will use the modern {@code
- * MemorySegment} API which allows to safely unmap (if you discover any problems with this preview
- * API, you can disable it by using system property {@link #ENABLE_MEMORY_SEGMENTS_SYSPROP}).
+ * <p>On exactly <b>Java 19 / 20 / 21</b> this class will use the modern {@code MemorySegment} API
+ * which allows to safely unmap (if you discover any problems with this preview API, you can disable
+ * it by using system property {@link #ENABLE_MEMORY_SEGMENTS_SYSPROP}).
*
* <p><b>NOTE:</b> Accessing this class either directly or indirectly from a thread while it's
* interrupted can close the underlying channel immediately if at the same time the thread is
@@ -123,7 +123,7 @@ public class MMapDirectory extends FSDirectory {
* Default max chunk size:
*
* <ul>
- * <li>16 GiBytes for 64 bit <b>Java 19</b> and <b>Java 20</b> JVMs
+ * <li>16 GiBytes for 64 bit <b>Java 19 / 20 / 21</b> JVMs
* <li>1 GiBytes for other 64 bit JVMs
* <li>256 MiBytes for 32 bit JVMs
* </ul>
@@ -346,7 +346,7 @@ public class MMapDirectory extends FSDirectory {
}
final var lookup = MethodHandles.lookup();
final int runtimeVersion = Runtime.version().feature();
- if (runtimeVersion == 19 || runtimeVersion == 20) {
+ if (runtimeVersion >= 19 && runtimeVersion <= 21) {
try {
final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider");
// we use method handles, so we do not need to deal with setAccessible as we have private
@@ -366,9 +366,9 @@ public class MMapDirectory extends FSDirectory {
throw new LinkageError(
"MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe);
}
- } else if (runtimeVersion >= 21) {
+ } else if (runtimeVersion >= 22) {
LOG.warning(
- "You are running with Java 21 or later. To make full use of MMapDirectory, please update Apache Lucene.");
+ "You are running with Java 22 or later. To make full use of MMapDirectory, please update Apache Lucene.");
}
return new MappedByteBufferIndexInputProvider();
}
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
new file mode 100644
index 00000000000..7b2216add78
--- /dev/null
+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
@@ -0,0 +1,588 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.store;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.Objects;
+import org.apache.lucene.util.ArrayUtil;
+
+/**
+ * Base IndexInput implementation that uses an array of MemorySegments to represent a file.
+ *
+ * <p>For efficiency, this class requires that the segment size are a power-of-two (<code>
+ * chunkSizePower</code>).
+ */
+@SuppressWarnings("preview")
+abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccessInput {
+ static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE;
+ static final ValueLayout.OfShort LAYOUT_LE_SHORT =
+ ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+ static final ValueLayout.OfInt LAYOUT_LE_INT =
+ ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+ static final ValueLayout.OfLong LAYOUT_LE_LONG =
+ ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+ static final ValueLayout.OfFloat LAYOUT_LE_FLOAT =
+ ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+
+ final long length;
+ final long chunkSizeMask;
+ final int chunkSizePower;
+ final Arena arena;
+ final MemorySegment[] segments;
+
+ int curSegmentIndex = -1;
+ MemorySegment
+ curSegment; // redundant for speed: segments[curSegmentIndex], also marker if closed!
+ long curPosition; // relative to curSegment, not globally
+
+ public static MemorySegmentIndexInput newInstance(
+ String resourceDescription,
+ Arena arena,
+ MemorySegment[] segments,
+ long length,
+ int chunkSizePower) {
+ assert Arrays.stream(segments).map(MemorySegment::scope).allMatch(arena.scope()::equals);
+ if (segments.length == 1) {
+ return new SingleSegmentImpl(resourceDescription, arena, segments[0], length, chunkSizePower);
+ } else {
+ return new MultiSegmentImpl(resourceDescription, arena, segments, 0, length, chunkSizePower);
+ }
+ }
+
+ private MemorySegmentIndexInput(
+ String resourceDescription,
+ Arena arena,
+ MemorySegment[] segments,
+ long length,
+ int chunkSizePower) {
+ super(resourceDescription);
+ this.arena = arena;
+ this.segments = segments;
+ this.length = length;
+ this.chunkSizePower = chunkSizePower;
+ this.chunkSizeMask = (1L << chunkSizePower) - 1L;
+ this.curSegment = segments[0];
+ }
+
+ void ensureOpen() {
+ if (curSegment == null) {
+ throw alreadyClosed(null);
+ }
+ }
+
+ // the unused parameter is just to silence javac about unused variables
+ RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
+ throws IOException {
+ if (pos < 0L) {
+ return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this);
+ } else {
+ throw new EOFException(action + " past EOF (pos=" + pos + "): " + this);
+ }
+ }
+
+ // the unused parameter is just to silence javac about unused variables
+ AlreadyClosedException alreadyClosed(RuntimeException unused) {
+ return new AlreadyClosedException("Already closed: " + this);
+ }
+
+ @Override
+ public final byte readByte() throws IOException {
+ try {
+ final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
+ curPosition++;
+ return v;
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException e) {
+ do {
+ curSegmentIndex++;
+ if (curSegmentIndex >= segments.length) {
+ throw new EOFException("read past EOF: " + this);
+ }
+ curSegment = segments[curSegmentIndex];
+ curPosition = 0L;
+ } while (curSegment.byteSize() == 0L);
+ final byte v = curSegment.get(LAYOUT_BYTE, curPosition);
+ curPosition++;
+ return v;
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public final void readBytes(byte[] b, int offset, int len) throws IOException {
+ try {
+ MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
+ curPosition += len;
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException e) {
+ readBytesBoundary(b, offset, len);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ private void readBytesBoundary(byte[] b, int offset, int len) throws IOException {
+ try {
+ long curAvail = curSegment.byteSize() - curPosition;
+ while (len > curAvail) {
+ MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, (int) curAvail);
+ len -= curAvail;
+ offset += curAvail;
+ curSegmentIndex++;
+ if (curSegmentIndex >= segments.length) {
+ throw new EOFException("read past EOF: " + this);
+ }
+ curSegment = segments[curSegmentIndex];
+ curPosition = 0L;
+ curAvail = curSegment.byteSize();
+ }
+ MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len);
+ curPosition += len;
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public void readInts(int[] dst, int offset, int length) throws IOException {
+ try {
+ MemorySegment.copy(curSegment, LAYOUT_LE_INT, curPosition, dst, offset, length);
+ curPosition += Integer.BYTES * (long) length;
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException iobe) {
+ super.readInts(dst, offset, length);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public void readLongs(long[] dst, int offset, int length) throws IOException {
+ try {
+ MemorySegment.copy(curSegment, LAYOUT_LE_LONG, curPosition, dst, offset, length);
+ curPosition += Long.BYTES * (long) length;
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException iobe) {
+ super.readLongs(dst, offset, length);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public void readFloats(float[] dst, int offset, int length) throws IOException {
+ try {
+ MemorySegment.copy(curSegment, LAYOUT_LE_FLOAT, curPosition, dst, offset, length);
+ curPosition += Float.BYTES * (long) length;
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException iobe) {
+ super.readFloats(dst, offset, length);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public final short readShort() throws IOException {
+ try {
+ final short v = curSegment.get(LAYOUT_LE_SHORT, curPosition);
+ curPosition += Short.BYTES;
+ return v;
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException e) {
+ return super.readShort();
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public final int readInt() throws IOException {
+ try {
+ final int v = curSegment.get(LAYOUT_LE_INT, curPosition);
+ curPosition += Integer.BYTES;
+ return v;
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException e) {
+ return super.readInt();
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public final long readLong() throws IOException {
+ try {
+ final long v = curSegment.get(LAYOUT_LE_LONG, curPosition);
+ curPosition += Long.BYTES;
+ return v;
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException e) {
+ return super.readLong();
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public long getFilePointer() {
+ ensureOpen();
+ return (((long) curSegmentIndex) << chunkSizePower) + curPosition;
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ ensureOpen();
+ // we use >> here to preserve negative, so we will catch AIOOBE,
+ // in case pos + offset overflows.
+ final int si = (int) (pos >> chunkSizePower);
+ try {
+ if (si != curSegmentIndex) {
+ final MemorySegment seg = segments[si];
+ // write values, on exception all is unchanged
+ this.curSegmentIndex = si;
+ this.curSegment = seg;
+ }
+ this.curPosition = Objects.checkIndex(pos & chunkSizeMask, curSegment.byteSize() + 1);
+ } catch (IndexOutOfBoundsException e) {
+ throw handlePositionalIOOBE(e, "seek", pos);
+ }
+ }
+
+ @Override
+ public byte readByte(long pos) throws IOException {
+ try {
+ final int si = (int) (pos >> chunkSizePower);
+ return segments[si].get(LAYOUT_BYTE, pos & chunkSizeMask);
+ } catch (IndexOutOfBoundsException ioobe) {
+ throw handlePositionalIOOBE(ioobe, "read", pos);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ // used only by random access methods to handle reads across boundaries
+ private void setPos(long pos, int si) throws IOException {
+ try {
+ final MemorySegment seg = segments[si];
+ // write values, on exception above all is unchanged
+ this.curPosition = pos & chunkSizeMask;
+ this.curSegmentIndex = si;
+ this.curSegment = seg;
+ } catch (IndexOutOfBoundsException ioobe) {
+ throw handlePositionalIOOBE(ioobe, "read", pos);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ final int si = (int) (pos >> chunkSizePower);
+ try {
+ return segments[si].get(LAYOUT_LE_SHORT, pos & chunkSizeMask);
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException ioobe) {
+ // either it's a boundary, or read past EOF, fall back:
+ setPos(pos, si);
+ return readShort();
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ final int si = (int) (pos >> chunkSizePower);
+ try {
+ return segments[si].get(LAYOUT_LE_INT, pos & chunkSizeMask);
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException ioobe) {
+ // either it's a boundary, or read past EOF, fall back:
+ setPos(pos, si);
+ return readInt();
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ final int si = (int) (pos >> chunkSizePower);
+ try {
+ return segments[si].get(LAYOUT_LE_LONG, pos & chunkSizeMask);
+ } catch (
+ @SuppressWarnings("unused")
+ IndexOutOfBoundsException ioobe) {
+ // either it's a boundary, or read past EOF, fall back:
+ setPos(pos, si);
+ return readLong();
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public final long length() {
+ return length;
+ }
+
+ @Override
+ public final MemorySegmentIndexInput clone() {
+ final MemorySegmentIndexInput clone = buildSlice((String) null, 0L, this.length);
+ try {
+ clone.seek(getFilePointer());
+ } catch (IOException ioe) {
+ throw new AssertionError(ioe);
+ }
+
+ return clone;
+ }
+
+ /**
+ * Creates a slice of this index input, with the given description, offset, and length. The slice
+ * is seeked to the beginning.
+ */
+ @Override
+ public final MemorySegmentIndexInput slice(String sliceDescription, long offset, long length) {
+ if (offset < 0 || length < 0 || offset + length > this.length) {
+ throw new IllegalArgumentException(
+ "slice() "
+ + sliceDescription
+ + " out of bounds: offset="
+ + offset
+ + ",length="
+ + length
+ + ",fileLength="
+ + this.length
+ + ": "
+ + this);
+ }
+
+ return buildSlice(sliceDescription, offset, length);
+ }
+
+ /** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */
+ MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) {
+ ensureOpen();
+
+ final long sliceEnd = offset + length;
+ final int startIndex = (int) (offset >>> chunkSizePower);
+ final int endIndex = (int) (sliceEnd >>> chunkSizePower);
+
+ // we always allocate one more slice, the last one may be a 0 byte one after truncating with
+ // asSlice():
+ final MemorySegment slices[] = ArrayUtil.copyOfSubArray(segments, startIndex, endIndex + 1);
+
+ // set the last segment's limit for the sliced view.
+ slices[slices.length - 1] = slices[slices.length - 1].asSlice(0L, sliceEnd & chunkSizeMask);
+
+ offset = offset & chunkSizeMask;
+
+ final String newResourceDescription = getFullSliceDescription(sliceDescription);
+ if (slices.length == 1) {
+ return new SingleSegmentImpl(
+ newResourceDescription,
+ null, // clones don't have an Arena, as they can't close)
+ slices[0].asSlice(offset, length),
+ length,
+ chunkSizePower);
+ } else {
+ return new MultiSegmentImpl(
+ newResourceDescription,
+ null, // clones don't have an Arena, as they can't close)
+ slices,
+ offset,
+ length,
+ chunkSizePower);
+ }
+ }
+
+ @Override
+ public final void close() throws IOException {
+ if (curSegment == null) {
+ return;
+ }
+
+ // make sure all accesses to this IndexInput instance throw NPE:
+ curSegment = null;
+ Arrays.fill(segments, null);
+
+ // the master IndexInput has an Arena and is able
+ // to release all resources (unmap segments) - a
+ // side effect is that other threads still using clones
+ // will throw IllegalStateException
+ if (arena != null) {
+ arena.close();
+ }
+ }
+
+ /** Optimization of MemorySegmentIndexInput for when there is only one segment. */
+ static final class SingleSegmentImpl extends MemorySegmentIndexInput {
+
+ SingleSegmentImpl(
+ String resourceDescription,
+ Arena arena,
+ MemorySegment segment,
+ long length,
+ int chunkSizePower) {
+ super(resourceDescription, arena, new MemorySegment[] {segment}, length, chunkSizePower);
+ this.curSegmentIndex = 0;
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ ensureOpen();
+ try {
+ curPosition = Objects.checkIndex(pos, length + 1);
+ } catch (IndexOutOfBoundsException e) {
+ throw handlePositionalIOOBE(e, "seek", pos);
+ }
+ }
+
+ @Override
+ public long getFilePointer() {
+ ensureOpen();
+ return curPosition;
+ }
+
+ @Override
+ public byte readByte(long pos) throws IOException {
+ try {
+ return curSegment.get(LAYOUT_BYTE, pos);
+ } catch (IndexOutOfBoundsException e) {
+ throw handlePositionalIOOBE(e, "read", pos);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ try {
+ return curSegment.get(LAYOUT_LE_SHORT, pos);
+ } catch (IndexOutOfBoundsException e) {
+ throw handlePositionalIOOBE(e, "read", pos);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ try {
+ return curSegment.get(LAYOUT_LE_INT, pos);
+ } catch (IndexOutOfBoundsException e) {
+ throw handlePositionalIOOBE(e, "read", pos);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ try {
+ return curSegment.get(LAYOUT_LE_LONG, pos);
+ } catch (IndexOutOfBoundsException e) {
+ throw handlePositionalIOOBE(e, "read", pos);
+ } catch (NullPointerException | IllegalStateException e) {
+ throw alreadyClosed(e);
+ }
+ }
+ }
+
+ /** This class adds offset support to MemorySegmentIndexInput, which is needed for slices. */
+ static final class MultiSegmentImpl extends MemorySegmentIndexInput {
+ private final long offset;
+
+ MultiSegmentImpl(
+ String resourceDescription,
+ Arena arena,
+ MemorySegment[] segments,
+ long offset,
+ long length,
+ int chunkSizePower) {
+ super(resourceDescription, arena, segments, length, chunkSizePower);
+ this.offset = offset;
+ try {
+ seek(0L);
+ } catch (IOException ioe) {
+ throw new AssertionError(ioe);
+ }
+ assert curSegment != null && curSegmentIndex >= 0;
+ }
+
+ @Override
+ RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos)
+ throws IOException {
+ return super.handlePositionalIOOBE(unused, action, pos - offset);
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ assert pos >= 0L : "negative position";
+ super.seek(pos + offset);
+ }
+
+ @Override
+ public long getFilePointer() {
+ return super.getFilePointer() - offset;
+ }
+
+ @Override
+ public byte readByte(long pos) throws IOException {
+ return super.readByte(pos + offset);
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ return super.readShort(pos + offset);
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ return super.readInt(pos + offset);
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ return super.readLong(pos + offset);
+ }
+
+ @Override
+ MemorySegmentIndexInput buildSlice(String sliceDescription, long ofs, long length) {
+ return super.buildSlice(sliceDescription, this.offset + ofs, length);
+ }
+ }
+}
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
new file mode 100644
index 00000000000..e994c2dddff
--- /dev/null
+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.store;
+
+import java.io.IOException;
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.logging.Logger;
+import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.Unwrappable;
+
+@SuppressWarnings("preview")
+final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider {
+
+ public MemorySegmentIndexInputProvider() {
+ var log = Logger.getLogger(getClass().getName());
+ log.info(
+ "Using MemorySegmentIndexInput with Java 21; to disable start with -D"
+ + MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP
+ + "=false");
+ }
+
+ @Override
+ public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload)
+ throws IOException {
+ final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")";
+
+ // Work around for JDK-8259028: we need to unwrap our test-only file system layers
+ path = Unwrappable.unwrapAll(path);
+
+ boolean success = false;
+ final Arena arena = Arena.ofShared();
+ try (var fc = FileChannel.open(path, StandardOpenOption.READ)) {
+ final long fileSize = fc.size();
+ final IndexInput in =
+ MemorySegmentIndexInput.newInstance(
+ resourceDescription,
+ arena,
+ map(arena, resourceDescription, fc, chunkSizePower, preload, fileSize),
+ fileSize,
+ chunkSizePower);
+ success = true;
+ return in;
+ } finally {
+ if (success == false) {
+ arena.close();
+ }
+ }
+ }
+
+ @Override
+ public long getDefaultMaxChunkSize() {
+ return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28);
+ }
+
+ @Override
+ public boolean isUnmapSupported() {
+ return true;
+ }
+
+ @Override
+ public String getUnmapNotSupportedReason() {
+ return null;
+ }
+
+ private final MemorySegment[] map(
+ Arena arena,
+ String resourceDescription,
+ FileChannel fc,
+ int chunkSizePower,
+ boolean preload,
+ long length)
+ throws IOException {
+ if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
+ throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);
+
+ final long chunkSize = 1L << chunkSizePower;
+
+ // we always allocate one more segments, the last one may be a 0 byte one
+ final int nrSegments = (int) (length >>> chunkSizePower) + 1;
+
+ final MemorySegment[] segments = new MemorySegment[nrSegments];
+
+ long startOffset = 0L;
+ for (int segNr = 0; segNr < nrSegments; segNr++) {
+ final long segSize =
+ (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset);
+ final MemorySegment segment;
+ try {
+ segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena);
+ } catch (IOException ioe) {
+ throw convertMapFailedIOException(ioe, resourceDescription, segSize);
+ }
+ if (preload) {
+ segment.load();
+ }
+ segments[segNr] = segment;
+ startOffset += segSize;
+ }
+ return segments;
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
index dc638f6a529..5597161a3a9 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
@@ -48,9 +48,9 @@ public class TestMmapDirectory extends BaseDirectoryTestCase {
public void testCorrectImplementation() {
final int runtimeVersion = Runtime.version().feature();
- if (runtimeVersion == 19 || runtimeVersion == 20) {
+ if (runtimeVersion >= 19 && runtimeVersion <= 21) {
assertTrue(
- "on Java 19 and Java 20 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
+ "on Java 19, 20, and 21 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs",
isMemorySegmentImpl());
} else {
assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass());
diff --git a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
index b3e55f27770..a5b8f6409e9 100644
--- a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
+++ b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java
@@ -206,7 +206,7 @@ public class TestModularLayer extends AbstractLuceneDistributionTest {
ClassLoader loader = layer.findLoader(coreModuleId);
- final Set<Integer> jarVersions = Set.of(19, 20);
+ final Set<Integer> jarVersions = Set.of(19, 20, 21);
for (var v : jarVersions) {
Assertions.assertThat(
loader.getResource(