You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/09/11 07:10:30 UTC
(paimon) branch master updated: [common] Version the memory-mode cache key in CachingFileIO (#9642)
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 099464e7ec [common] Version the memory-mode cache key in CachingFileIO (#9642)
099464e7ec is described below
commit 099464e7ec17bae65af40ae2725b6d3b8e58b746
Author: YangJie <ya...@baidu.com>
AuthorDate: Fri Sep 11 03:10:24 2026 -0400
[common] Version the memory-mode cache key in CachingFileIO (#9642)
---
.../java/org/apache/paimon/utils/FileType.java | 14 +++++-
.../apache/paimon/fs/cache/CachingFileIOTest.java | 57 ++++++++++++++++++++++
.../java/org/apache/paimon/utils/FileTypeTest.java | 32 ++++++++++++
3 files changed, 101 insertions(+), 2 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java b/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
index 46c767bb24..9b30c864c1 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
@@ -114,8 +114,18 @@ public enum FileType {
/** Returns {@code true} if the file is mutable and should not be cached. */
public static boolean isMutable(Path filePath) {
- String name = filePath.getName();
- return "EARLIEST".equals(name) || "LATEST".equals(name);
+ String name = unwrapTempFileName(filePath.getName());
+ // Files rewritten in place under a stable path: caching them by path keeps serving the
+ // pre-overwrite content (and a len+mtime key still collides when a rewrite lands at the
+ // same size within the same clock second). Hint files, consumer and service progress
+ // files, replaceable tags and _SUCCESS all go through overwriteFileUtf8.
+ return "EARLIEST".equals(name)
+ || "LATEST".equals(name)
+ || "_SUCCESS".equals(name)
+ || name.endsWith("_SUCCESS")
+ || name.startsWith(CONSUMER_PREFIX)
+ || name.startsWith(SERVICE_PREFIX)
+ || name.startsWith(TAG_PREFIX);
}
/**
diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java
index ded23667a9..05ddd7cad0 100644
--- a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java
@@ -94,6 +94,63 @@ class CachingFileIOTest {
MockFileIO.resetGlobalInputStreamCalls();
}
+ @Test
+ void testMemoryModeDoesNotCacheInPlaceOverwrittenFiles() throws IOException {
+ MockFileIO delegate = new MockFileIO();
+ CachingFileIO cachingIO =
+ newCachingFileIO(
+ delegate,
+ new LocalMemoryCacheManager(Long.MAX_VALUE, 64),
+ EnumSet.of(FileType.META),
+ 64);
+ Path consumer = new Path("consumer-1");
+
+ // consumer-* is written in place by overwriteFileUtf8, so it is mutable and bypasses the
+ // cache: each read reaches the delegate and sees the current content.
+ delegate.addFile("consumer-1", "v1cc".getBytes());
+ try (SeekableInputStream in = cachingIO.newInputStream(consumer)) {
+ assertThat(in).isNotInstanceOf(CachingSeekableInputStream.class);
+ byte[] buf = new byte[4];
+ in.read(buf, 0, 4);
+ assertThat(new String(buf)).isEqualTo("v1cc");
+ }
+ assertThat(delegate.newInputStreamCallCount("consumer-1")).isEqualTo(1);
+
+ delegate.addFile("consumer-1", "v2cc".getBytes());
+ try (SeekableInputStream in = cachingIO.newInputStream(consumer)) {
+ byte[] buf = new byte[4];
+ in.read(buf, 0, 4);
+ assertThat(new String(buf)).isEqualTo("v2cc");
+ }
+ // never cached: the overwrite is visible and the delegate was opened again
+ assertThat(delegate.newInputStreamCallCount("consumer-1")).isEqualTo(2);
+ }
+
+ @Test
+ void testMemoryModeImmutableCacheHitsDoNotRestatDelegate() throws IOException {
+ MockFileIO delegate = new MockFileIO();
+ CachingFileIO cachingIO =
+ newCachingFileIO(
+ delegate,
+ new LocalMemoryCacheManager(Long.MAX_VALUE, 64),
+ EnumSet.of(FileType.META),
+ 64);
+ Path snapshot = new Path("snapshot-1");
+ delegate.addFile("snapshot-1", "0123456789abcdef".getBytes());
+
+ for (int i = 0; i < 3; i++) {
+ try (SeekableInputStream in = cachingIO.newInputStream(snapshot)) {
+ assertThat(readAll(in, 16)).isEqualTo("0123456789abcdef".getBytes());
+ }
+ }
+
+ // An immutable file keeps the path-only memory key: opened once, then served from cache.
+ // Its size is resolved lazily and remembered, so repeated hits do not re-stat the
+ // delegate the way moving getFileStatus onto every open would.
+ assertThat(delegate.newInputStreamCallCount("snapshot-1")).isEqualTo(1);
+ assertThat(delegate.getFileStatusCallCount("snapshot-1")).isEqualTo(1);
+ }
+
@Test
void testCreateBlobPresignedUrlDelegates() throws IOException {
FileIO delegate = mock(FileIO.class);
diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
index d946f71bb7..d043d6bf97 100644
--- a/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
@@ -31,6 +31,38 @@ public class FileTypeTest {
private static final String TABLE_ROOT = "hdfs://cluster/warehouse/db.db/table";
+ // ===== mutable files (bypass the cache) =====
+
+ @Test
+ public void testIsMutable() {
+ // Files overwritten in place under a stable path must bypass the cache.
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/snapshot/EARLIEST"))).isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/snapshot/LATEST"))).isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/consumer/consumer-myGroup")))
+ .isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/service/service-primary-key-lookup")))
+ .isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/tag/tag-myTag"))).isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/dt=2024-01-01/bucket-0/_SUCCESS")))
+ .isTrue();
+ // A temp rewrite of a mutable file is still mutable.
+ assertThat(
+ FileType.isMutable(
+ new Path(
+ TABLE_ROOT
+ + "/consumer/.consumer-myGroup."
+ + UUID.randomUUID()
+ + ".tmp")))
+ .isTrue();
+
+ // Write-once files stay cacheable: a new version lands under a new name.
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/snapshot/snapshot-1"))).isFalse();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/schema/schema-0"))).isFalse();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/manifest/manifest-a1b2c3d4-0")))
+ .isFalse();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT + "/bucket-0/data-abc.orc"))).isFalse();
+ }
+
// ===== META files =====
@Test