You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2022/11/15 09:54:47 UTC
[lucene] branch branch_9x updated: More granular control of preloading on MMapDirectory. (#11929)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new d2418d3e876 More granular control of preloading on MMapDirectory. (#11929)
d2418d3e876 is described below
commit d2418d3e876c785ce6b1a9b7ca62166827ef0a63
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Nov 15 10:52:49 2022 +0100
More granular control of preloading on MMapDirectory. (#11929)
This enables configuring preloading on MMapDirectory based on the file name as well as the IOContext that is used to open the file.
---
lucene/CHANGES.txt | 3 ++
.../org/apache/lucene/store/MMapDirectory.java | 54 ++++++++++++++++++----
.../org/apache/lucene/store/TestMmapDirectory.java | 2 +-
3 files changed, 50 insertions(+), 9 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ece36dcf2e4..22b74cb1878 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -25,6 +25,9 @@ New Features
* GITHUB#11795: Add ByteWritesTrackingDirectoryWrapper to expose metrics for bytes merged, flushed, and overall
write amplification factor. (Marc D'Mello)
+* GITHUB#11929: MMapDirectory gives more granular control on which files to
+ preload. (Adrien Grand, Uwe Schindler)
+
Improvements
---------------------
* GITHUB#11785: Improve Tessellator performance by delaying calls to the method
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index 80517ac00f6..a6c70b7e612 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -23,6 +23,7 @@ import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.Path;
import java.util.Locale;
import java.util.concurrent.Future;
+import java.util.function.BiPredicate;
import java.util.logging.Logger;
import org.apache.lucene.util.Constants;
@@ -38,6 +39,10 @@ import org.apache.lucene.util.Constants;
* fragmented address space. If you get an OutOfMemoryException, it is recommended to reduce the
* chunk size, until it works.
*
+ * <p>This class supports preloading files into physical memory upon opening. This can help improve
+ * performance of searches on a cold page cache at the expense of slowing down opening an index. See
+ * {@link #setPreload(BiPredicate)} for more details.
+ *
* <p>Due to <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">this bug</a> in
* Sun's JRE, MMapDirectory's {@link IndexInput#close} is unable to close the underlying OS file
* handle. Only when GC finally collects the underlying objects, which could be quite some time
@@ -74,8 +79,21 @@ import org.apache.lucene.util.Constants;
* about MMapDirectory</a>
*/
public class MMapDirectory extends FSDirectory {
+
+ /**
+ * Argument for {@link #setPreload(BiPredicate)} that configures all files to be preloaded upon
+ * opening them.
+ */
+ public static final BiPredicate<String, IOContext> ALL_FILES = (filename, context) -> true;
+
+ /**
+ * Argument for {@link #setPreload(BiPredicate)} that configures no files to be preloaded upon
+ * opening them.
+ */
+ public static final BiPredicate<String, IOContext> NO_FILES = (filename, context) -> false;
+
private boolean useUnmapHack = UNMAP_SUPPORTED;
- private boolean preload;
+ private BiPredicate<String, IOContext> preload = NO_FILES;
/**
* Default max chunk size:
@@ -226,20 +244,39 @@ public class MMapDirectory extends FSDirectory {
}
/**
- * Set to {@code true} to ask mapped pages to be loaded into physical memory on init. The behavior
- * is best-effort and operating system dependent.
+ * Configure which files to preload in physical memory upon opening. The default implementation
+ * does not preload anything. The behavior is best effort and operating system-dependent.
+ *
+ * @param preload a {@link BiPredicate} whose first argument is the file name, and second argument
+ * is the {@link IOContext} used to open the file
+ * @see #ALL_FILES
+ * @see #NO_FILES
+ */
+ public void setPreload(BiPredicate<String, IOContext> preload) {
+ this.preload = preload;
+ }
+
+ /**
+ * Configure whether to preload files on this {@link MMapDirectory} into physical memory upon
+ * opening. The behavior is best effort and operating system-dependent.
+ *
+ * @deprecated Use {@link #setPreload(BiPredicate)} instead which provides more granular control.
*/
+ @Deprecated
public void setPreload(boolean preload) {
- this.preload = preload;
+ this.preload = preload ? ALL_FILES : NO_FILES;
}
/**
- * Returns {@code true} if mapped pages should be loaded.
+ * Return whether files are loaded into physical memory upon opening.
*
- * @see #setPreload
+ * @deprecated This information is no longer reliable now that preloading is more granularly
+ * configured via a predicate.
+ * @see #setPreload(BiPredicate)
*/
+ @Deprecated
public boolean getPreload() {
- return preload;
+ return preload == ALL_FILES;
}
/**
@@ -257,7 +294,8 @@ public class MMapDirectory extends FSDirectory {
ensureOpen();
ensureCanRead(name);
Path path = directory.resolve(name);
- return PROVIDER.openInput(path, context, chunkSizePower, preload, useUnmapHack);
+ return PROVIDER.openInput(
+ path, context, chunkSizePower, preload.test(name, context), useUnmapHack);
}
// visible for tests:
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
index 810f8578e80..579b0c27108 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java
@@ -32,7 +32,7 @@ public class TestMmapDirectory extends BaseDirectoryTestCase {
@Override
protected Directory getDirectory(Path path) throws IOException {
MMapDirectory m = new MMapDirectory(path);
- m.setPreload(random().nextBoolean());
+ m.setPreload((file, context) -> random().nextBoolean());
return m;
}