You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by yu...@apache.org on 2026/08/30 10:46:54 UTC

(fluss) branch main updated: [server] Remove lookup lock for `PaimonLakeTableLookuper` (#4050)

This is an automated email from the ASF dual-hosted git repository.

luoyuxia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git


The following commit(s) were added to refs/heads/main by this push:
     new 9f6a698f4 [server] Remove lookup lock for `PaimonLakeTableLookuper` (#4050)
9f6a698f4 is described below

commit 9f6a698f42b6b508854cc72bdc900758aebeaa79
Author: Junfan Zhang <zu...@apache.org>
AuthorDate: Sun Aug 30 18:46:49 2026 +0800

    [server] Remove lookup lock for `PaimonLakeTableLookuper` (#4050)
---
 .../paimon/lookup/PaimonLakeTableLookuper.java     | 391 ++++++++++-----------
 .../paimon/lookup/PaimonLakeTableLookuperTest.java |  94 ++++-
 2 files changed, 273 insertions(+), 212 deletions(-)

diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java
index 01d29aac3..7723cac10 100644
--- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java
+++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java
@@ -25,7 +25,6 @@ import org.apache.fluss.exception.KvStorageException;
 import org.apache.fluss.lake.lakestorage.LakeTableLookuper;
 import org.apache.fluss.lake.paimon.utils.PaimonPartitionBucket;
 import org.apache.fluss.lake.paimon.utils.PaimonRowAsFlussRow;
-import org.apache.fluss.metadata.ResolvedPartitionSpec;
 import org.apache.fluss.metadata.TablePath;
 import org.apache.fluss.row.BinaryRow;
 import org.apache.fluss.row.InternalRow;
@@ -62,10 +61,10 @@ import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Set;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2;
 import static org.apache.fluss.lake.paimon.PaimonLakeCatalog.LEGACY_SYSTEM_COLUMNS;
@@ -85,11 +84,15 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull;
  * <p>A cached partition-bucket file set can become stale when Paimon compaction replaces its data
  * files and snapshot expiration physically deletes the old files. Because {@code FileIO}
  * implementations may represent a missing file with different {@link IOException} types, the first
- * lookup I/O failure closes the cached query state, reopens the table from the latest snapshot, and
+ * lookup I/O failure refreshes that partition-bucket with the files from the latest snapshot and
  * retries once.
  *
- * <p>Lookup and close operations are synchronized because they share mutable Paimon query, local
- * cache, and value-encoding state.
+ * <p>Lookup concurrency is delegated to {@link LocalTableQuery}. Older Paimon versions may
+ * serialize lookups internally, while Paimon 2.0 supports concurrent lookups without an additional
+ * Fluss-level lock.
+ *
+ * <p>Close is expected only after the owner has drained active lookups. It is synchronized with
+ * lazy initialization, but deliberately does not add a lifecycle lock to every lookup.
  */
 public class PaimonLakeTableLookuper implements LakeTableLookuper {
 
@@ -100,26 +103,22 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
     private final long lookupCacheMaxDiskBytes;
     private final Runnable diskWriteGuard;
 
-    private final Set<PaimonPartitionBucket> initializedBuckets;
+    private final ThreadLocal<Boolean> lookupFileDownloaded;
+    private final Object initializationLock;
+    private final Map<PaimonPartitionBucket, List<DataFileMeta>> registeredFiles;
 
     private @Nullable Catalog catalog;
     private @Nullable FileStoreTable fileStoreTable;
     private @Nullable IOManager ioManager;
-    private @Nullable LocalTableQuery localTableQuery;
-    private @Nullable RowPartitionKeyExtractor partitionKeyExtractor;
-    private int primaryKeyFieldCount;
-    private long lookupFileDownloadCount;
-
-    // Both encoders are initialized only for a kv-format-v2 table whose bucket key differs from
-    // its physical primary key. They remain null when the incoming Fluss key already uses Paimon's
-    // BinaryRow encoding and no conversion is needed.
+    private @Nullable List<String> trimmedPrimaryKeys;
+
+    // CompactedKeyDecoder contains immutable type metadata and creates all decode state per
+    // invocation, so it can be shared by concurrent lookups.
     private @Nullable CompactedKeyDecoder compactedKeyDecoder;
-    private @Nullable PaimonKeyEncoder paimonKeyEncoder;
-    private boolean hasCachedValueEncoder;
-    private short cachedValueSchemaId;
-    private @Nullable RowEncoder cachedValueRowEncoder;
-    private @Nullable InternalRow.FieldGetter[] cachedValueFieldGetters;
-    private boolean closed;
+
+    private volatile @Nullable LocalTableQuery localTableQuery;
+    // Guarded by initializationLock.
+    private volatile boolean closed;
 
     /** Creates a lookuper with the specified local lookup cache limit. */
     public PaimonLakeTableLookuper(
@@ -137,29 +136,20 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
                 lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0.");
         this.lookupCacheMaxDiskBytes = lookupCacheMaxDiskBytes;
         this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null.");
-        this.initializedBuckets = new HashSet<>();
+        this.lookupFileDownloaded = new ThreadLocal<>();
+        this.initializationLock = new Object();
+        this.registeredFiles = new ConcurrentHashMap<>();
     }
 
     @Override
-    public synchronized @Nullable byte[] lookup(byte[] key, LookupContext context)
-            throws Exception {
+    public @Nullable byte[] lookup(byte[] key, LookupContext context) throws Exception {
         checkNotNull(key, "key must not be null.");
         checkNotNull(context, "context must not be null.");
         checkNotClosed();
         ensureInitialized(context.valueRowType());
 
-        org.apache.paimon.data.BinaryRow partition =
-                convertPartition(context.partitionSpec(), context.valueRowType());
-        org.apache.paimon.data.BinaryRow keyRow = toPaimonLookupKey(key);
-        initializeFilesIfNeeded(partition, context.bucketId());
-
-        long downloadCountBeforeLookup = lookupFileDownloadCount;
-        long lookupStartNanos = System.nanoTime();
-        org.apache.paimon.data.InternalRow paimonRow;
-        try {
-            paimonRow =
-                    lookupWithFileRefresh(
-                            partition, context.bucketId(), keyRow, context.valueRowType());
+        try (TrackingMetrics ignored = new TrackingMetrics(lookupFileDownloaded, context)) {
+            return lookupInternal(key, context);
         } catch (Exception e) {
             DiskWriteLockedException diskWriteLockedException =
                     ExceptionUtils.findThrowable(e, DiskWriteLockedException.class).orElse(null);
@@ -167,31 +157,27 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
                 throw diskWriteLockedException;
             }
             throw e;
-        } finally {
-            context.lookupMetricRecorder()
-                    .recordLookup(
-                            System.nanoTime() - lookupStartNanos,
-                            // An increase means this lookup downloaded at least one lookup file
-                            // through the tracking IO manager.
-                            lookupFileDownloadCount > downloadCountBeforeLookup);
         }
-        if (paimonRow == null) {
-            return null;
-        }
-        return encodeValue(paimonRow, context.schemaId(), context.valueRowType());
     }
 
     @Override
-    public synchronized void close() {
-        if (closed) {
-            return;
+    public void close() {
+        synchronized (initializationLock) {
+            if (closed) {
+                return;
+            }
+            closed = true;
+            IOUtils.closeQuietly(localTableQuery, "Paimon lookup engine");
+            IOUtils.closeQuietly(ioManager, "Paimon lookup IO manager");
+            IOUtils.closeQuietly(catalog, "Paimon catalog");
+            registeredFiles.clear();
+            localTableQuery = null;
+            compactedKeyDecoder = null;
+            trimmedPrimaryKeys = null;
+            ioManager = null;
+            fileStoreTable = null;
+            catalog = null;
         }
-        closed = true;
-        IOUtils.closeQuietly(cachedValueRowEncoder, "Fluss value row encoder");
-        IOUtils.closeQuietly(localTableQuery, "Paimon local table query");
-        IOUtils.closeQuietly(ioManager, "Paimon lookup IO manager");
-        IOUtils.closeQuietly(catalog, "Paimon catalog");
-        initializedBuckets.clear();
     }
 
     private void checkNotClosed() {
@@ -201,10 +187,16 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
     }
 
     private void ensureInitialized(RowType valueRowType) throws Exception {
-        if (localTableQuery != null) {
-            return;
+        if (localTableQuery == null) {
+            synchronized (initializationLock) {
+                if (localTableQuery == null) {
+                    initialize(valueRowType);
+                }
+            }
         }
+    }
 
+    private void initialize(RowType valueRowType) throws Exception {
         Catalog newCatalog = null;
         IOManager newIOManager = null;
         LocalTableQuery newLocalTableQuery = null;
@@ -221,40 +213,36 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
                         "Point lookup is only supported for primary-key Paimon tables.");
             }
 
-            newIOManager = createIOManager(ioTmpDir);
-            newLocalTableQuery = newFileStoreTable.newLocalTableQuery();
-            newLocalTableQuery.withValueProjection(businessFieldProjection(newFileStoreTable));
-            newLocalTableQuery.withIOManager(newIOManager);
-            RowPartitionKeyExtractor newPartitionKeyExtractor =
-                    new RowPartitionKeyExtractor(newFileStoreTable.schema());
-            List<String> trimmedPrimaryKeys = newFileStoreTable.schema().trimmedPrimaryKeys();
-            int newPrimaryKeyFieldCount = trimmedPrimaryKeys.size();
+            List<String> newTrimmedPrimaryKeys =
+                    Collections.unmodifiableList(
+                            new ArrayList<>(newFileStoreTable.schema().trimmedPrimaryKeys()));
             CompactedKeyDecoder newCompactedKeyDecoder = null;
-            PaimonKeyEncoder newPaimonKeyEncoder = null;
 
             // Legacy/v1 tables and v2 tables with a default bucket key already encode Fluss
             // lookup keys with Paimon's key encoder. Only v2 tables with a non-default bucket
             // key use the compacted key encoding and need conversion before querying Paimon.
             if (tableConfig.getKvFormatVersion().orElse(1) == KV_FORMAT_VERSION_2
-                    && !newFileStoreTable.schema().bucketKeys().equals(trimmedPrimaryKeys)) {
+                    && !newFileStoreTable.schema().bucketKeys().equals(newTrimmedPrimaryKeys)) {
                 // Kv-format-v2 tables with a non-default bucket key store Fluss keys using the
                 // compacted encoding to support prefix lookup. Paimon's LocalTableQuery expects
                 // its own BinaryRow encoding, so convert the key at the lake lookup boundary.
                 newCompactedKeyDecoder =
-                        CompactedKeyDecoder.createKeyDecoder(valueRowType, trimmedPrimaryKeys);
-                RowType keyRowType = valueRowType.project(trimmedPrimaryKeys);
-                newPaimonKeyEncoder = new PaimonKeyEncoder(keyRowType, trimmedPrimaryKeys);
+                        CompactedKeyDecoder.createKeyDecoder(valueRowType, newTrimmedPrimaryKeys);
             }
 
-            // Publish the newly created state only after every initialization step succeeds.
+            newIOManager = createIOManager(ioTmpDir);
+            newLocalTableQuery =
+                    newFileStoreTable
+                            .newLocalTableQuery()
+                            .withValueProjection(businessFieldProjection(newFileStoreTable))
+                            .withIOManager(newIOManager);
+
             catalog = newCatalog;
             fileStoreTable = newFileStoreTable;
             ioManager = newIOManager;
-            partitionKeyExtractor = newPartitionKeyExtractor;
-            primaryKeyFieldCount = newPrimaryKeyFieldCount;
+            trimmedPrimaryKeys = newTrimmedPrimaryKeys;
             compactedKeyDecoder = newCompactedKeyDecoder;
-            paimonKeyEncoder = newPaimonKeyEncoder;
-            // localTableQuery is the initialization marker, so publish it last.
+            // Keep this volatile write last to publish all initialized fields together.
             localTableQuery = newLocalTableQuery;
             initialized = true;
         } finally {
@@ -292,139 +280,134 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
         return projection;
     }
 
-    private org.apache.paimon.data.BinaryRow convertPartition(
-            ResolvedPartitionSpec partitionSpec, RowType valueRowType) {
-        // The generated partition projection reuses its mutable output, while lookup caches retain
-        // the returned row as a hash key. Copy it before it escapes to keep those keys stable.
-        return toPaimonPartition(
-                        partitionSpec,
-                        valueRowType,
-                        fileStoreTable().schema().logicalRowType(),
-                        partitionKeyExtractor()::partition)
-                .copy();
+    private org.apache.paimon.data.BinaryRow getPartition(LookupContext context) {
+        // Both generated helpers reuse mutable writers or projections, so keep them confined to
+        // this lookup call.
+        RowPartitionKeyExtractor partitionKeyExtractor =
+                new RowPartitionKeyExtractor(fileStoreTable.schema());
+        org.apache.paimon.data.BinaryRow partition =
+                toPaimonPartition(
+                                context.partitionSpec(),
+                                context.valueRowType(),
+                                fileStoreTable.schema().logicalRowType(),
+                                partitionKeyExtractor::partition)
+                        .copy();
+        return partition;
     }
 
-    private org.apache.paimon.data.BinaryRow toPaimonLookupKey(byte[] key) {
+    private org.apache.paimon.data.BinaryRow getKey(byte[] key, LookupContext context) {
         byte[] paimonKey = key;
         if (compactedKeyDecoder != null) {
-            // A non-null decoder means the Fluss lookup key uses compacted encoding. Decode it
-            // first, then re-encode it as the Paimon BinaryRow expected by LocalTableQuery.
             InternalRow decodedKey = compactedKeyDecoder.decodeKey(key);
-            paimonKey =
-                    checkNotNull(paimonKeyEncoder, "Paimon key encoder must be initialized.")
-                            .encodeKey(decodedKey);
+            RowType keyRowType = context.valueRowType().project(trimmedPrimaryKeys);
+            PaimonKeyEncoder paimonKeyEncoder =
+                    new PaimonKeyEncoder(keyRowType, trimmedPrimaryKeys);
+            paimonKey = paimonKeyEncoder.encodeKey(decodedKey);
         }
 
         org.apache.paimon.data.BinaryRow keyRow =
-                new org.apache.paimon.data.BinaryRow(primaryKeyFieldCount);
+                new org.apache.paimon.data.BinaryRow(trimmedPrimaryKeys.size());
         keyRow.pointTo(MemorySegment.wrap(paimonKey), 0, paimonKey.length);
         return keyRow;
     }
 
-    private void initializeFilesIfNeeded(org.apache.paimon.data.BinaryRow partition, int bucketId) {
-        PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucketId);
-        if (initializedBuckets.contains(partitionBucket)) {
-            return;
+    private @Nullable byte[] lookupInternal(byte[] key, LookupContext context) {
+        org.apache.paimon.data.InternalRow paimonRow;
+        try {
+            paimonRow =
+                    lookupPaimon(getPartition(context), context.bucketId(), getKey(key, context));
+        } catch (IOException e) {
+            // Historical Paimon point lookup is part of the Fluss KV lookup path. Expose a
+            // persistent I/O failure as a retriable KV error so the existing KV RPC retry
+            // semantics can handle it consistently.
+            throw new KvStorageException(
+                    "Failed to lookup historical data from Paimon after refreshing files for "
+                            + tablePath
+                            + ".",
+                    e);
         }
-
-        LinkedHashMap<String, DataFileMeta> dataFilesByName = new LinkedHashMap<>();
-
-        InnerTableScan tableScan =
-                fileStoreTable()
-                        .newScan()
-                        .withPartitionFilter(Collections.singletonList(partition))
-                        .withBucket(bucketId);
-        for (Split split : tableScan.plan().splits()) {
-            if (!(split instanceof DataSplit)) {
-                continue;
-            }
-            DataSplit dataSplit = (DataSplit) split;
-            addFilesByName(dataFilesByName, dataSplit.dataFiles());
+        if (paimonRow == null) {
+            return null;
         }
-
-        // TODO: Refresh the file set if writes to expired partitions are supported in the future.
-        // Historical lookup is triggered only after the original Fluss partition has expired and
-        // been dropped. This PR does not support writes to expired partitions, so no new rows are
-        // expected and initializing the file set once is sufficient. Compaction-related missing
-        // files are handled by the IOException refresh path below.
-        // This partition-bucket has no registered lookup levels yet, so there are no old files to
-        // remove when building its lookup state from the active data files.
-        localTableQuery()
-                .refreshFiles(
-                        partition,
-                        bucketId,
-                        Collections.emptyList(),
-                        new ArrayList<>(dataFilesByName.values()));
-        initializedBuckets.add(partitionBucket);
+        return encodeValue(paimonRow, context.schemaId(), context.valueRowType());
     }
 
-    private org.apache.paimon.data.InternalRow lookupWithFileRefresh(
+    private @Nullable org.apache.paimon.data.InternalRow lookupPaimon(
             org.apache.paimon.data.BinaryRow partition,
-            int bucketId,
-            org.apache.paimon.data.InternalRow keyRow,
-            RowType valueRowType)
-            throws Exception {
+            int bucket,
+            org.apache.paimon.data.InternalRow key)
+            throws IOException {
+        List<DataFileMeta> filesBeforeLookup = initializeFiles(partition, bucket);
         try {
-            return localTableQuery().lookup(partition, bucketId, keyRow);
-        } catch (IOException e) {
-            // FileIO only guarantees IOException and storage plugins may use different exception
-            // types for a missing file. The missing old file after compaction may therefore
-            // surface as any IOException. Refresh and retry only once so persistent I/O failures
-            // do not repeatedly rebuild Paimon lookup state within one request.
+            return localTableQuery.lookup(partition, bucket, key);
+        } catch (IOException firstError) {
+            refreshFilesIfUnchanged(partition, bucket, filesBeforeLookup);
             try {
-                refreshFiles(partition, bucketId, valueRowType);
-                return localTableQuery().lookup(partition, bucketId, keyRow);
+                return localTableQuery.lookup(partition, bucket, key);
             } catch (IOException retryError) {
-                retryError.addSuppressed(e);
-                // Historical Paimon point lookup is part of the Fluss KV lookup path. Expose a
-                // persistent I/O failure as a retriable KV error so the existing KV RPC retry
-                // semantics can handle it consistently.
-                throw new KvStorageException(
-                        "Failed to lookup historical data from Paimon after refreshing files for "
-                                + tablePath
-                                + ".",
-                        retryError);
+                retryError.addSuppressed(firstError);
+                throw retryError;
             }
         }
     }
 
-    private void refreshFiles(
-            org.apache.paimon.data.BinaryRow partition, int bucketId, RowType valueRowType)
-            throws Exception {
-        IOUtils.closeQuietly(localTableQuery, "Paimon local table query");
-        IOUtils.closeQuietly(ioManager, "Paimon lookup IO manager");
-        IOUtils.closeQuietly(catalog, "Paimon catalog");
-        localTableQuery = null;
-        ioManager = null;
-        catalog = null;
-        fileStoreTable = null;
-        partitionKeyExtractor = null;
-        primaryKeyFieldCount = 0;
-        compactedKeyDecoder = null;
-        paimonKeyEncoder = null;
-        initializedBuckets.clear();
-        ensureInitialized(valueRowType);
-        initializeFilesIfNeeded(partition, bucketId);
+    private List<DataFileMeta> initializeFiles(
+            org.apache.paimon.data.BinaryRow partition, int bucket) {
+        PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucket);
+        return registeredFiles.computeIfAbsent(
+                partitionBucket,
+                ignored -> scanAndUpdateFiles(partition, bucket, Collections.emptyList()));
+    }
+
+    private void refreshFilesIfUnchanged(
+            org.apache.paimon.data.BinaryRow partition,
+            int bucket,
+            List<DataFileMeta> filesBeforeLookup) {
+        PaimonPartitionBucket partitionBucket = new PaimonPartitionBucket(partition, bucket);
+        registeredFiles.compute(
+                partitionBucket,
+                (ignored, currentFiles) -> {
+                    List<DataFileMeta> files =
+                            checkNotNull(
+                                    currentFiles, "Partition-bucket files must be initialized.");
+                    return files == filesBeforeLookup
+                            ? scanAndUpdateFiles(partition, bucket, filesBeforeLookup)
+                            : files;
+                });
     }
 
-    private static void addFilesByName(
-            LinkedHashMap<String, DataFileMeta> filesByName, List<DataFileMeta> files) {
-        for (DataFileMeta file : files) {
-            filesByName.put(file.fileName(), file);
+    private List<DataFileMeta> scanAndUpdateFiles(
+            org.apache.paimon.data.BinaryRow partition,
+            int bucket,
+            List<DataFileMeta> filesBeforeRefresh) {
+        List<DataFileMeta> latestFiles = scanDataFiles(partition, bucket);
+        localTableQuery.refreshFiles(partition, bucket, filesBeforeRefresh, latestFiles);
+        return latestFiles;
+    }
+
+    private List<DataFileMeta> scanDataFiles(
+            org.apache.paimon.data.BinaryRow partition, int bucket) {
+        LinkedHashMap<String, DataFileMeta> dataFilesByName = new LinkedHashMap<>();
+        InnerTableScan tableScan =
+                fileStoreTable
+                        .newScan()
+                        .withPartitionFilter(Collections.singletonList(partition))
+                        .withBucket(bucket);
+        for (Split split : tableScan.plan().splits()) {
+            if (split instanceof DataSplit) {
+                for (DataFileMeta file : ((DataSplit) split).dataFiles()) {
+                    dataFilesByName.put(file.fileName(), file);
+                }
+            }
         }
+        return Collections.unmodifiableList(new ArrayList<>(dataFilesByName.values()));
     }
 
     private byte[] encodeValue(
             org.apache.paimon.data.InternalRow paimonRow, short schemaId, RowType valueRowType) {
         PaimonRowAsFlussRow flussRow = new PaimonRowAsFlussRow(paimonRow);
-        try {
-            ensureValueEncoder(schemaId, valueRowType);
-            RowEncoder rowEncoder =
-                    checkNotNull(cachedValueRowEncoder, "cachedValueRowEncoder must not be null.");
-            InternalRow.FieldGetter[] fieldGetters =
-                    checkNotNull(
-                            cachedValueFieldGetters, "cachedValueFieldGetters must not be null.");
-
+        InternalRow.FieldGetter[] fieldGetters = InternalRow.createFieldGetters(valueRowType);
+        try (RowEncoder rowEncoder = RowEncoder.create(tableConfig.getKvFormat(), valueRowType)) {
             rowEncoder.startNewRow();
             for (int i = 0; i < fieldGetters.length; i++) {
                 rowEncoder.encodeField(i, fieldGetters[i].getFieldOrNull(flussRow));
@@ -436,30 +419,6 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
         }
     }
 
-    private void ensureValueEncoder(short schemaId, RowType valueRowType) {
-        if (hasCachedValueEncoder && cachedValueSchemaId == schemaId) {
-            return;
-        }
-
-        IOUtils.closeQuietly(cachedValueRowEncoder, "Fluss value row encoder");
-        cachedValueRowEncoder = RowEncoder.create(tableConfig.getKvFormat(), valueRowType);
-        cachedValueFieldGetters = InternalRow.createFieldGetters(valueRowType);
-        cachedValueSchemaId = schemaId;
-        hasCachedValueEncoder = true;
-    }
-
-    private FileStoreTable fileStoreTable() {
-        return checkNotNull(fileStoreTable, "fileStoreTable must be initialized.");
-    }
-
-    private LocalTableQuery localTableQuery() {
-        return checkNotNull(localTableQuery, "localTableQuery must be initialized.");
-    }
-
-    private RowPartitionKeyExtractor partitionKeyExtractor() {
-        return checkNotNull(partitionKeyExtractor, "partitionKeyExtractor must be initialized.");
-    }
-
     /** Tracks creation of Paimon lookup files while delegating all local I/O operations. */
     private final class TrackingIOManager implements IOManager {
 
@@ -483,8 +442,13 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
                 // I/O boundary here and unwrap the retriable Fluss exception in lookup().
                 throw new UncheckedIOException(new IOException(e));
             }
-            lookupFileDownloadCount++;
-            return delegate.createChannel(prefix);
+            FileIOChannel.ID channel = delegate.createChannel(prefix);
+            // Paimon creates lookup files synchronously in the lookup thread, so this marks only
+            // the request that caused this channel to be created.
+            if (lookupFileDownloaded.get() != null) {
+                lookupFileDownloaded.set(true);
+            }
+            return channel;
         }
 
         @Override
@@ -519,4 +483,25 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper {
             delegate.close();
         }
     }
+
+    private static final class TrackingMetrics implements AutoCloseable {
+        private final ThreadLocal<Boolean> lookupFileDownloaded;
+        private final long startNanoTime;
+        private final LookupContext context;
+
+        private TrackingMetrics(ThreadLocal<Boolean> lookupFileDownloaded, LookupContext context) {
+            this.lookupFileDownloaded = lookupFileDownloaded;
+            this.lookupFileDownloaded.set(false);
+            this.startNanoTime = System.nanoTime();
+            this.context = context;
+        }
+
+        @Override
+        public void close() throws Exception {
+            boolean fileDownloaded = lookupFileDownloaded.get();
+            lookupFileDownloaded.remove();
+            context.lookupMetricRecorder()
+                    .recordLookup(System.nanoTime() - startNanoTime, fileDownloaded);
+        }
+    }
 }
diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java
index 1bf0ec3a1..8d1a4f919 100644
--- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java
+++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java
@@ -38,6 +38,7 @@ import org.apache.fluss.row.encode.CompactedKeyEncoder;
 import org.apache.fluss.row.encode.ValueDecoder;
 import org.apache.fluss.row.encode.paimon.PaimonKeyEncoder;
 import org.apache.fluss.types.DataTypes;
+import org.apache.fluss.utils.ExecutorUtils;
 
 import org.apache.paimon.CoreOptions;
 import org.apache.paimon.catalog.Catalog;
@@ -64,6 +65,11 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2;
@@ -419,11 +425,31 @@ class PaimonLakeTableLookuperTest {
                     Collections.singletonMap(
                             0, Collections.singletonList(paimonRow(id, "20240101", "name-" + id))));
         }
+        writeAndCommitData(
+                table,
+                Collections.singletonMap(
+                        0, Collections.singletonList(paimonRow(6, "20240102", "name-6"))));
 
         BinaryRow partition = BinaryRow.singleColumn(BinaryString.fromString("20240101"));
         List<DataFileMeta> filesBeforeCompaction = dataFiles(table, partition, 0);
         assertThat(filesBeforeCompaction).hasSize(5);
 
+        AtomicBoolean blockDownloads = new AtomicBoolean();
+        CountDownLatch downloadStarted = new CountDownLatch(1);
+        CountDownLatch continueDownload = new CountDownLatch(1);
+        Runnable diskWriteGuard =
+                () -> {
+                    if (blockDownloads.get()) {
+                        downloadStarted.countDown();
+                        try {
+                            continueDownload.await();
+                        } catch (InterruptedException e) {
+                            Thread.currentThread().interrupt();
+                            throw new RuntimeException(e);
+                        }
+                    }
+                };
+
         try (LakeTableLookuper lookuper =
                 new PaimonLakeTableLookuper(
                         paimonConfig,
@@ -431,10 +457,17 @@ class PaimonLakeTableLookuperTest {
                         tempWarehouseDir.getAbsolutePath(),
                         tableConfig(KvFormat.COMPACTED),
                         LOOKUP_CACHE_MAX_DISK_BYTES,
-                        NO_OP_DISK_WRITE_GUARD)) {
-            LakeTableLookuper.LookupContext context =
-                    lookupContext(schema, "20240101", 0, SCHEMA_ID);
-            assertThat(lookuper.lookup(paimonKey(schema, 5, "20240101"), context)).isNotNull();
+                        diskWriteGuard)) {
+            assertThat(
+                            lookuper.lookup(
+                                    paimonKey(schema, 5, "20240101"),
+                                    lookupContext(schema, "20240101", 0, SCHEMA_ID)))
+                    .isNotNull();
+            assertThat(
+                            lookuper.lookup(
+                                    paimonKey(schema, 6, "20240102"),
+                                    lookupContext(schema, "20240102", 0, SCHEMA_ID)))
+                    .isNotNull();
 
             new CompactHelper(table, new File(tempWarehouseDir, "compact"))
                     .compactBucket(partition, 0)
@@ -462,12 +495,55 @@ class PaimonLakeTableLookuperTest {
                 assertThat(table.store().snapshotManager().fileIO().exists(path)).isFalse();
             }
 
-            BinaryValue decodedValue =
-                    decodeValue(
-                            lookuper.lookup(paimonKey(schema, 1, "20240101"), context),
+            List<Boolean> refreshedLookupDownloads = new ArrayList<>();
+            List<Boolean> cachedLookupDownloads = new ArrayList<>();
+            LakeTableLookuper.LookupContext refreshedContext =
+                    lookupContext(
+                            schema,
+                            "20240101",
+                            0,
                             SCHEMA_ID,
-                            schema);
-            assertRow(decodedValue.row, 1, "20240101", "name-1");
+                            (lookupTimeNanos, lookupFileDownloaded) ->
+                                    refreshedLookupDownloads.add(lookupFileDownloaded));
+            LakeTableLookuper.LookupContext cachedContext =
+                    lookupContext(
+                            schema,
+                            "20240102",
+                            0,
+                            SCHEMA_ID,
+                            (lookupTimeNanos, lookupFileDownloaded) ->
+                                    cachedLookupDownloads.add(lookupFileDownloaded));
+            ExecutorService executor = Executors.newFixedThreadPool(2);
+            blockDownloads.set(true);
+            try {
+                Future<byte[]> refreshedLookup =
+                        executor.submit(
+                                () ->
+                                        lookuper.lookup(
+                                                paimonKey(schema, 1, "20240101"),
+                                                refreshedContext));
+                assertThat(downloadStarted.await(30, TimeUnit.SECONDS)).isTrue();
+
+                Future<byte[]> cachedLookup =
+                        executor.submit(
+                                () ->
+                                        lookuper.lookup(
+                                                paimonKey(schema, 6, "20240102"), cachedContext));
+                BinaryValue cachedValue =
+                        decodeValue(cachedLookup.get(30, TimeUnit.SECONDS), SCHEMA_ID, schema);
+                assertRow(cachedValue.row, 6, "20240102", "name-6");
+
+                continueDownload.countDown();
+                BinaryValue refreshedValue =
+                        decodeValue(refreshedLookup.get(30, TimeUnit.SECONDS), SCHEMA_ID, schema);
+                assertRow(refreshedValue.row, 1, "20240101", "name-1");
+            } finally {
+                continueDownload.countDown();
+                ExecutorUtils.gracefulShutdown(30, TimeUnit.SECONDS, executor);
+            }
+
+            assertThat(refreshedLookupDownloads).containsExactly(true);
+            assertThat(cachedLookupDownloads).containsExactly(false);
         }
     }