You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ch...@apache.org on 2024/11/29 10:28:40 UTC
(lucene) branch main updated: Add IndexInput isLoaded (#13998)
This is an automated email from the ASF dual-hosted git repository.
chegar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 7dbbd0daa97 Add IndexInput isLoaded (#13998)
7dbbd0daa97 is described below
commit 7dbbd0daa97e1ffd0c8dd27d1441c257cf8751da
Author: Chris Hegarty <62...@users.noreply.github.com>
AuthorDate: Fri Nov 29 10:28:32 2024 +0000
Add IndexInput isLoaded (#13998)
This commit adds IndexInput::isLoaded to help determine if the contents of an input is resident in physical memory.
The intent of this new method is to help build inspection and diagnostic infrastructure on top.
---
.../java/org/apache/lucene/store/IndexInput.java | 16 +++++++++
.../org/apache/lucene/store/RandomAccessInput.java | 10 ++++++
.../lucene/store/MemorySegmentIndexInput.java | 10 ++++++
.../lucene/tests/store/BaseDirectoryTestCase.java | 42 ++++++++++++++++++++++
.../lucene/tests/store/MockIndexInputWrapper.java | 8 +++++
.../tests/store/SerialIOCountingDirectory.java | 6 ++++
6 files changed, 92 insertions(+)
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
index 09f9211ac17..b649ace8c4d 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Optional;
import org.apache.lucene.codecs.CompoundFormat;
/**
@@ -234,4 +235,19 @@ public abstract class IndexInput extends DataInput implements Closeable {
* <p>The default implementation is a no-op.
*/
public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {}
+
+ /**
+ * Returns a hint whether all the contents of this input are resident in physical memory. It's a
+ * hint because the operating system may have paged out some of the data by the time this method
+ * returns. If the optional is true, then it's likely that the contents of this input are resident
+ * in physical memory. A value of false does not imply that the contents are not resident in
+ * physical memory. An empty optional is returned if it is not possible to determine.
+ *
+ * <p>This runs in linear time with the {@link #length()} of this input / page size.
+ *
+ * <p>The default implementation returns an empty optional.
+ */
+ public Optional<Boolean> isLoaded() {
+ return Optional.empty();
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java b/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java
index 08b2e83d36d..44127d90a98 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RandomAccessInput.java
@@ -17,6 +17,7 @@
package org.apache.lucene.store;
import java.io.IOException;
+import java.util.Optional;
import org.apache.lucene.util.BitUtil; // javadocs
/**
@@ -77,4 +78,13 @@ public interface RandomAccessInput {
* @see IndexInput#prefetch
*/
default void prefetch(long offset, long length) throws IOException {}
+
+ /**
+ * Returns a hint whether all the contents of this input are resident in physical memory.
+ *
+ * @see IndexInput#isLoaded()
+ */
+ default Optional<Boolean> isLoaded() {
+ return Optional.empty();
+ }
}
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
index 5824e78bdc7..2fc9f6369b7 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
@@ -420,6 +420,16 @@ abstract class MemorySegmentIndexInput extends IndexInput
}
}
+ @Override
+ public Optional<Boolean> isLoaded() {
+ for (MemorySegment seg : segments) {
+ if (seg.isLoaded() == false) {
+ return Optional.of(Boolean.FALSE);
+ }
+ }
+ return Optional.of(Boolean.TRUE);
+ }
+
@Override
public byte readByte(long pos) throws IOException {
try {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
index e041afdbcb3..d8ec0529b5f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
@@ -51,9 +51,11 @@ import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.tests.mockfile.ExtrasFS;
@@ -1636,4 +1638,44 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
}
}
}
+
+ public void testIsLoaded() throws IOException {
+ testIsLoaded(0);
+ }
+
+ public void testIsLoadedOnSlice() throws IOException {
+ testIsLoaded(TestUtil.nextInt(random(), 1, 1024));
+ }
+
+ private void testIsLoaded(int startOffset) throws IOException {
+ try (Directory dir = getDirectory(createTempDir())) {
+ if (FilterDirectory.unwrap(dir) instanceof MMapDirectory mMapDirectory) {
+ mMapDirectory.setPreload(MMapDirectory.ALL_FILES);
+ }
+ final int totalLength = startOffset + TestUtil.nextInt(random(), 16384, 65536);
+ byte[] arr = new byte[totalLength];
+ random().nextBytes(arr);
+ try (IndexOutput out = dir.createOutput("temp.bin", IOContext.DEFAULT)) {
+ out.writeBytes(arr, arr.length);
+ }
+
+ try (IndexInput orig = dir.openInput("temp.bin", IOContext.DEFAULT)) {
+ IndexInput in;
+ if (startOffset == 0) {
+ in = orig.clone();
+ } else {
+ in = orig.slice("slice", startOffset, totalLength - startOffset);
+ }
+ var loaded = in.isLoaded();
+ if (FilterDirectory.unwrap(dir) instanceof MMapDirectory
+ // direct IO wraps MMap but does not support isLoaded
+ && !(dir.getClass().getName().contains("DirectIO"))) {
+ assertTrue(loaded.isPresent());
+ assertTrue(loaded.get());
+ } else {
+ assertFalse(loaded.isPresent());
+ }
+ }
+ }
+ }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
index aa5c841bf3b..1dc5456d64e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java
@@ -19,6 +19,7 @@ package org.apache.lucene.tests.store;
import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.lucene.internal.tests.TestSecrets;
import org.apache.lucene.store.FilterIndexInput;
@@ -184,6 +185,13 @@ public class MockIndexInputWrapper extends FilterIndexInput {
in.prefetch(offset, length);
}
+ @Override
+ public Optional<Boolean> isLoaded() {
+ ensureOpen();
+ ensureAccessible();
+ return in.isLoaded();
+ }
+
@Override
public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {
ensureOpen();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java
index 2afbc2fd4a7..1b4234c3d79 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/SerialIOCountingDirectory.java
@@ -17,6 +17,7 @@
package org.apache.lucene.tests.store;
import java.io.IOException;
+import java.util.Optional;
import java.util.concurrent.atomic.LongAdder;
import org.apache.lucene.internal.hppc.LongHashSet;
import org.apache.lucene.store.ChecksumIndexInput;
@@ -206,5 +207,10 @@ public class SerialIOCountingDirectory extends FilterDirectory {
IndexInput clone = in.clone();
return new SerializedIOCountingIndexInput(clone, readAdvice, sliceOffset, sliceLength);
}
+
+ @Override
+ public Optional<Boolean> isLoaded() {
+ return in.isLoaded();
+ }
}
}