You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/01/10 14:05:11 UTC

(incubator-paimon) 02/06: [flink] Refactor FileStoreLookupFunction to abstract more into LookupTable

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git

commit b13d99f4bfb43034ca86459010e34da6926a0df2
Author: Jingsong <ji...@gmail.com>
AuthorDate: Tue Jan 9 14:53:46 2024 +0800

    [flink] Refactor FileStoreLookupFunction to abstract more into LookupTable
---
 .../generated/flink_connector_configuration.html   |   4 +-
 .../apache/paimon/predicate/PredicateFilter.java   |   5 +-
 .../apache/paimon/flink/FlinkConnectorOptions.java |   6 +-
 .../flink/lookup/FileStoreLookupFunction.java      | 130 +++---------
 ...ractor.java => FixedBucketFromPkExtractor.java} |   4 +-
 .../paimon/flink/lookup/FullCacheLookupTable.java  | 172 +++++++++++++++-
 .../apache/paimon/flink/lookup/LookupTable.java    |  82 +-------
 .../flink/lookup/NoPrimaryKeyLookupTable.java      |  35 +---
 .../flink/lookup/PartialCacheLookupTable.java      |  30 ---
 .../paimon/flink/lookup/PrimaryKeyLookupTable.java |  59 ++----
 .../flink/lookup/PrimaryKeyPartialLookupTable.java |  42 +++-
 .../flink/lookup/SecondaryIndexLookupTable.java    |  49 ++---
 .../paimon/flink/source/DataTableSource.java       |   8 -
 .../org/apache/paimon/flink/LookupJoinITCase.java  |  87 ++------
 .../paimon/flink/lookup/LookupTableTest.java       | 223 ++++++++++++---------
 15 files changed, 432 insertions(+), 504 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/flink_connector_configuration.html b/docs/layouts/shortcodes/generated/flink_connector_configuration.html
index 5e407fa53..099d9127c 100644
--- a/docs/layouts/shortcodes/generated/flink_connector_configuration.html
+++ b/docs/layouts/shortcodes/generated/flink_connector_configuration.html
@@ -52,9 +52,9 @@ under the License.
         </tr>
         <tr>
             <td><h5>lookup.cache</h5></td>
-            <td style="word-wrap: break-word;">FULL</td>
+            <td style="word-wrap: break-word;">AUTO</td>
             <td><p>Enum</p></td>
-            <td>The cache mode of lookup join.<br /><br />Possible values:<ul><li>"PARTIAL"</li><li>"FULL"</li></ul></td>
+            <td>The cache mode of lookup join.<br /><br />Possible values:<ul><li>"AUTO"</li><li>"FULL"</li></ul></td>
         </tr>
         <tr>
             <td><h5>scan.infer-parallelism</h5></td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateFilter.java b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateFilter.java
index 900214c42..f88bc1445 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateFilter.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/PredicateFilter.java
@@ -20,14 +20,15 @@ package org.apache.paimon.predicate;
 
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Filter;
 import org.apache.paimon.utils.RowDataToObjectArrayConverter;
 
 import javax.annotation.Nullable;
 
 import java.util.List;
 
-/** A {@link java.util.function.Predicate} to filter {@link InternalRow}. */
-public class PredicateFilter implements java.util.function.Predicate<InternalRow> {
+/** A {@link Filter} to filter {@link InternalRow}. */
+public class PredicateFilter implements Filter<InternalRow> {
 
     private final RowDataToObjectArrayConverter arrayConverter;
     @Nullable private final Predicate predicate;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
index ea41831e7..ae6451916 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java
@@ -290,7 +290,7 @@ public class FlinkConnectorOptions {
     public static final ConfigOption<LookupCacheMode> LOOKUP_CACHE_MODE =
             ConfigOptions.key("lookup.cache")
                     .enumType(LookupCacheMode.class)
-                    .defaultValue(LookupCacheMode.FULL)
+                    .defaultValue(LookupCacheMode.AUTO)
                     .withDescription("The cache mode of lookup join.");
 
     public static final ConfigOption<Boolean> SINK_AUTO_TAG_FOR_SAVEPOINT =
@@ -331,8 +331,8 @@ public class FlinkConnectorOptions {
 
     /** The mode of lookup cache. */
     public enum LookupCacheMode {
-        /** Use partial caching mode. */
-        PARTIAL,
+        /** Auto mode, try to use partial mode. */
+        AUTO,
 
         /** Use full caching mode. */
         FULL
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
index 27fd0b2dd..96b7b0b47 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java
@@ -18,32 +18,18 @@
 
 package org.apache.paimon.flink.lookup;
 
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.GenericRow;
 import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.disk.IOManager;
-import org.apache.paimon.flink.FlinkConnectorOptions;
 import org.apache.paimon.flink.FlinkConnectorOptions.LookupCacheMode;
 import org.apache.paimon.flink.FlinkRowData;
 import org.apache.paimon.flink.FlinkRowWrapper;
-import org.apache.paimon.flink.lookup.FullCacheLookupTable.TableBulkLoader;
 import org.apache.paimon.flink.utils.TableScanUtils;
-import org.apache.paimon.lookup.BulkLoader;
-import org.apache.paimon.lookup.RocksDBState;
-import org.apache.paimon.lookup.RocksDBStateFactory;
 import org.apache.paimon.options.Options;
 import org.apache.paimon.predicate.Predicate;
 import org.apache.paimon.predicate.PredicateFilter;
-import org.apache.paimon.reader.RecordReaderIterator;
-import org.apache.paimon.sort.BinaryExternalSortBuffer;
+import org.apache.paimon.table.FileStoreTable;
 import org.apache.paimon.table.Table;
 import org.apache.paimon.table.source.OutOfRangeException;
-import org.apache.paimon.types.DataTypes;
-import org.apache.paimon.types.RowType;
 import org.apache.paimon.utils.FileIOUtils;
-import org.apache.paimon.utils.MutableObjectIterator;
-import org.apache.paimon.utils.PartialRow;
 import org.apache.paimon.utils.TypeUtils;
 
 import org.apache.paimon.shade.guava30.com.google.common.primitives.Ints;
@@ -66,6 +52,7 @@ import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
@@ -73,10 +60,10 @@ import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static org.apache.paimon.CoreOptions.CONTINUOUS_DISCOVERY_INTERVAL;
+import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_CACHE_MODE;
 import static org.apache.paimon.lookup.RocksDBOptions.LOOKUP_CACHE_ROWS;
 import static org.apache.paimon.lookup.RocksDBOptions.LOOKUP_CONTINUOUS_DISCOVERY_INTERVAL;
 import static org.apache.paimon.predicate.PredicateBuilder.transformFieldMapping;
-import static org.apache.paimon.schema.SystemColumns.SEQUENCE_NUMBER;
 
 /** A lookup {@link TableFunction} for file store. */
 public class FileStoreLookupFunction implements Serializable, Closeable {
@@ -92,16 +79,10 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
 
     private transient Duration refreshInterval;
     private transient File path;
-    private transient RocksDBStateFactory stateFactory;
     private transient LookupTable lookupTable;
 
     // timestamp when cache expires
     private transient long nextLoadTime;
-    private transient TableStreamingReader streamingReader;
-
-    private final boolean sequenceFieldEnabled;
-
-    private transient TableFileMonitor fileMonitor;
 
     public FileStoreLookupFunction(
             Table table, int[] projection, int[] joinKeyIndex, @Nullable Predicate predicate) {
@@ -128,9 +109,6 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
         }
 
         this.predicate = predicate;
-        this.sequenceFieldEnabled =
-                table.primaryKeys().size() > 0
-                        && new CoreOptions(table.options()).sequenceField().isPresent();
     }
 
     public void open(FunctionContext context) throws Exception {
@@ -148,79 +126,39 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
     }
 
     private void open() throws Exception {
+        this.nextLoadTime = -1;
+
         Options options = Options.fromMap(table.options());
         this.refreshInterval =
                 options.getOptional(LOOKUP_CONTINUOUS_DISCOVERY_INTERVAL)
                         .orElse(options.get(CONTINUOUS_DISCOVERY_INTERVAL));
-        LookupCacheMode cacheMode = options.get(FlinkConnectorOptions.LOOKUP_CACHE_MODE);
 
         List<String> fieldNames = table.rowType().getFieldNames();
         int[] projection = projectFields.stream().mapToInt(fieldNames::indexOf).toArray();
-        RowType rowType = TypeUtils.project(table.rowType(), projection);
+        FileStoreTable storeTable = (FileStoreTable) table;
 
-        PredicateFilter recordFilter = createRecordFilter(projection);
-
-        switch (cacheMode) {
-            case FULL:
-                this.stateFactory = new RocksDBStateFactory(path.toString(), options, null);
+        if (options.get(LOOKUP_CACHE_MODE) == LookupCacheMode.AUTO
+                && new HashSet<>(table.primaryKeys()).equals(new HashSet<>(joinKeys))) {
+            try {
                 this.lookupTable =
-                        LookupTable.create(
-                                stateFactory,
-                                sequenceFieldEnabled
-                                        ? rowType.appendDataField(
-                                                SEQUENCE_NUMBER, DataTypes.BIGINT())
-                                        : rowType,
-                                table.primaryKeys(),
-                                joinKeys,
-                                recordFilter,
-                                options.get(LOOKUP_CACHE_ROWS));
-                this.streamingReader = new TableStreamingReader(table, projection, this.predicate);
-                bulkLoad(options);
-                break;
-            case PARTIAL:
-                this.lookupTable = LookupTable.create(table, projection, joinKeys, path);
-                this.fileMonitor = new TableFileMonitor(table, this.predicate);
-                ((PartialCacheLookupTable) lookupTable).refresh(fileMonitor.readChanges());
-                break;
-            default:
-                throw new IllegalArgumentException("Unsupported lookup cache mode: " + cacheMode);
-        }
-        this.nextLoadTime = -1;
-    }
-
-    private void bulkLoad(Options options) throws Exception {
-        BinaryExternalSortBuffer bulkLoadSorter =
-                RocksDBState.createBulkLoadSorter(
-                        IOManager.create(path.toString()), new CoreOptions(options));
-        FullCacheLookupTable lookupTable = (FullCacheLookupTable) this.lookupTable;
-        try (RecordReaderIterator<InternalRow> batch =
-                new RecordReaderIterator<>(streamingReader.nextBatch(true, sequenceFieldEnabled))) {
-            while (batch.hasNext()) {
-                InternalRow row = batch.next();
-                if (lookupTable.recordFilter().test(row)) {
-                    bulkLoadSorter.write(
-                            GenericRow.of(
-                                    lookupTable.toKeyBytes(row), lookupTable.toValueBytes(row)));
-                }
+                        new PrimaryKeyPartialLookupTable(storeTable, predicate, projection, path);
+            } catch (UnsupportedOperationException ignore) {
             }
         }
 
-        MutableObjectIterator<BinaryRow> keyIterator = bulkLoadSorter.sortedIterator();
-        BinaryRow row = new BinaryRow(2);
-        TableBulkLoader bulkLoader = lookupTable.createBulkLoader();
-        try {
-            while ((row = keyIterator.next(row)) != null) {
-                bulkLoader.write(row.getBinary(0), row.getBinary(1));
-            }
-        } catch (BulkLoader.WriteException e) {
-            throw new RuntimeException(
-                    "Exception in bulkLoad, the most suspicious reason is that "
-                            + "your data contains duplicates, please check your lookup table. ",
-                    e.getCause());
+        if (lookupTable == null) {
+            FullCacheLookupTable.Context context =
+                    new FullCacheLookupTable.Context(
+                            storeTable,
+                            projection,
+                            predicate,
+                            path,
+                            createRecordFilter(projection),
+                            joinKeys);
+            this.lookupTable = FullCacheLookupTable.create(context, options.get(LOOKUP_CACHE_ROWS));
         }
 
-        bulkLoader.finish();
-        bulkLoadSorter.clear();
+        lookupTable.open();
     }
 
     private PredicateFilter createRecordFilter(int[] projection) {
@@ -245,9 +183,6 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
             List<InternalRow> results = lookupTable.get(new FlinkRowWrapper(keyRow));
             List<RowData> rows = new ArrayList<>(results.size());
             for (InternalRow matchedRow : results) {
-                if (sequenceFieldEnabled) {
-                    matchedRow = new PartialRow(matchedRow.getFieldCount() - 1, matchedRow);
-                }
                 rows.add(new FlinkRowData(matchedRow));
             }
             return rows;
@@ -285,27 +220,14 @@ public class FileStoreLookupFunction implements Serializable, Closeable {
     }
 
     private void refresh() throws Exception {
-        if (lookupTable instanceof FullCacheLookupTable) {
-            while (true) {
-                try (RecordReaderIterator<InternalRow> batch =
-                        new RecordReaderIterator<>(
-                                streamingReader.nextBatch(false, sequenceFieldEnabled))) {
-                    if (!batch.hasNext()) {
-                        return;
-                    }
-                    ((FullCacheLookupTable) this.lookupTable).refresh(batch, sequenceFieldEnabled);
-                }
-            }
-        } else {
-            ((PartialCacheLookupTable) lookupTable).refresh(fileMonitor.readChanges());
-        }
+        lookupTable.refresh();
     }
 
     @Override
     public void close() throws IOException {
-        if (stateFactory != null) {
-            stateFactory.close();
-            stateFactory = null;
+        if (lookupTable != null) {
+            lookupTable.close();
+            lookupTable = null;
         }
 
         if (path != null) {
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketKeyExtractor.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketFromPkExtractor.java
similarity index 96%
rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketKeyExtractor.java
rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketFromPkExtractor.java
index 07e524a26..2c296992a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketKeyExtractor.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FixedBucketFromPkExtractor.java
@@ -29,7 +29,7 @@ import org.apache.paimon.table.sink.KeyAndBucketExtractor;
 import static org.apache.paimon.utils.Preconditions.checkArgument;
 
 /** Extractor to extract bucket from the primary key. */
-public class FixedBucketKeyExtractor implements KeyAndBucketExtractor<InternalRow> {
+public class FixedBucketFromPkExtractor implements KeyAndBucketExtractor<InternalRow> {
 
     private transient InternalRow primaryKey;
 
@@ -47,7 +47,7 @@ public class FixedBucketKeyExtractor implements KeyAndBucketExtractor<InternalRo
 
     private final Projection logPrimaryKeyProjection;
 
-    public FixedBucketKeyExtractor(TableSchema schema) {
+    public FixedBucketFromPkExtractor(TableSchema schema) {
         this.numBuckets = new CoreOptions(schema.options()).bucket();
         checkArgument(numBuckets > 0, "Num bucket is illegal: " + numBuckets);
         this.sameBucketKeyAndTrimmedPrimaryKey =
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
index b9988e1ca..ed4451053 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java
@@ -20,31 +20,187 @@
 
 package org.apache.paimon.flink.lookup;
 
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericRow;
 import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.disk.IOManager;
 import org.apache.paimon.lookup.BulkLoader;
+import org.apache.paimon.lookup.RocksDBState;
+import org.apache.paimon.lookup.RocksDBStateFactory;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.reader.RecordReaderIterator;
+import org.apache.paimon.sort.BinaryExternalSortBuffer;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Filter;
+import org.apache.paimon.utils.MutableObjectIterator;
+import org.apache.paimon.utils.PartialRow;
+import org.apache.paimon.utils.TypeUtils;
 
+import javax.annotation.Nullable;
+
+import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Iterator;
-import java.util.function.Predicate;
+import java.util.List;
+
+import static org.apache.paimon.schema.SystemColumns.SEQUENCE_NUMBER;
 
 /** Lookup table of full cache. */
-public interface FullCacheLookupTable extends LookupTable {
+public abstract class FullCacheLookupTable implements LookupTable {
+
+    protected final Context context;
+    protected final RocksDBStateFactory stateFactory;
+    protected final RowType projectedType;
+    private final TableStreamingReader reader;
+    private final boolean sequenceFieldEnabled;
+
+    public FullCacheLookupTable(Context context) throws IOException {
+        this.context = context;
+        this.stateFactory =
+                new RocksDBStateFactory(
+                        context.tempPath.toString(),
+                        context.table.coreOptions().toConfiguration(),
+                        null);
+        FileStoreTable table = context.table;
+        this.reader = new TableStreamingReader(table, context.projection, context.predicate);
+        this.sequenceFieldEnabled =
+                table.primaryKeys().size() > 0
+                        && new CoreOptions(table.options()).sequenceField().isPresent();
+        RowType projectedType = TypeUtils.project(table.rowType(), context.projection);
+        if (sequenceFieldEnabled) {
+            projectedType = projectedType.appendDataField(SEQUENCE_NUMBER, DataTypes.BIGINT());
+        }
+        this.projectedType = projectedType;
+    }
+
+    @Override
+    public void open() throws Exception {
+        BinaryExternalSortBuffer bulkLoadSorter =
+                RocksDBState.createBulkLoadSorter(
+                        IOManager.create(context.tempPath.toString()), context.table.coreOptions());
+        try (RecordReaderIterator<InternalRow> batch =
+                new RecordReaderIterator<>(reader.nextBatch(true, sequenceFieldEnabled))) {
+            while (batch.hasNext()) {
+                InternalRow row = batch.next();
+                if (recordFilter().test(row)) {
+                    bulkLoadSorter.write(GenericRow.of(toKeyBytes(row), toValueBytes(row)));
+                }
+            }
+        }
+
+        MutableObjectIterator<BinaryRow> keyIterator = bulkLoadSorter.sortedIterator();
+        BinaryRow row = new BinaryRow(2);
+        TableBulkLoader bulkLoader = createBulkLoader();
+        try {
+            while ((row = keyIterator.next(row)) != null) {
+                bulkLoader.write(row.getBinary(0), row.getBinary(1));
+            }
+        } catch (BulkLoader.WriteException e) {
+            throw new RuntimeException(
+                    "Exception in bulkLoad, the most suspicious reason is that "
+                            + "your data contains duplicates, please check your lookup table. ",
+                    e.getCause());
+        }
+
+        bulkLoader.finish();
+        bulkLoadSorter.clear();
+    }
+
+    public void refresh() throws Exception {
+        while (true) {
+            try (RecordReaderIterator<InternalRow> batch =
+                    new RecordReaderIterator<>(reader.nextBatch(false, sequenceFieldEnabled))) {
+                if (!batch.hasNext()) {
+                    return;
+                }
+                refresh(batch, sequenceFieldEnabled);
+            }
+        }
+    }
+
+    @Override
+    public final List<InternalRow> get(InternalRow key) throws IOException {
+        List<InternalRow> values = innerGet(key);
+        if (!sequenceFieldEnabled) {
+            return values;
+        }
 
-    void refresh(Iterator<InternalRow> input, boolean orderByLastField) throws IOException;
+        List<InternalRow> dropSequence = new ArrayList<>(values.size());
+        for (InternalRow matchedRow : values) {
+            dropSequence.add(new PartialRow(matchedRow.getFieldCount() - 1, matchedRow));
+        }
+        return dropSequence;
+    }
+
+    public abstract List<InternalRow> innerGet(InternalRow key) throws IOException;
+
+    public abstract void refresh(Iterator<InternalRow> input, boolean orderByLastField)
+            throws IOException;
 
-    Predicate<InternalRow> recordFilter();
+    public Filter<InternalRow> recordFilter() {
+        return context.recordFilter;
+    }
+
+    public abstract byte[] toKeyBytes(InternalRow row) throws IOException;
 
-    byte[] toKeyBytes(InternalRow row) throws IOException;
+    public abstract byte[] toValueBytes(InternalRow row) throws IOException;
 
-    byte[] toValueBytes(InternalRow row) throws IOException;
+    public abstract TableBulkLoader createBulkLoader();
 
-    TableBulkLoader createBulkLoader();
+    @Override
+    public void close() throws IOException {
+        stateFactory.close();
+    }
 
     /** Bulk loader for the table. */
-    interface TableBulkLoader {
+    public interface TableBulkLoader {
 
         void write(byte[] key, byte[] value) throws BulkLoader.WriteException, IOException;
 
         void finish() throws IOException;
     }
+
+    static FullCacheLookupTable create(Context context, long lruCacheSize) throws IOException {
+        List<String> primaryKeys = context.table.primaryKeys();
+        if (primaryKeys.isEmpty()) {
+            return new NoPrimaryKeyLookupTable(context, lruCacheSize);
+        } else {
+            if (new HashSet<>(primaryKeys).equals(new HashSet<>(context.joinKey))) {
+                return new PrimaryKeyLookupTable(context, lruCacheSize);
+            } else {
+                return new SecondaryIndexLookupTable(context, lruCacheSize);
+            }
+        }
+    }
+
+    /** Context for {@link LookupTable}. */
+    public static class Context {
+
+        public final FileStoreTable table;
+        public final int[] projection;
+        public final @Nullable Predicate predicate;
+        public final File tempPath;
+        public final Filter<InternalRow> recordFilter;
+        public final List<String> joinKey;
+
+        public Context(
+                FileStoreTable table,
+                int[] projection,
+                @Nullable Predicate predicate,
+                File tempPath,
+                Filter<InternalRow> recordFilter,
+                List<String> joinKey) {
+            this.table = table;
+            this.projection = projection;
+            this.predicate = predicate;
+            this.tempPath = tempPath;
+            this.recordFilter = recordFilter;
+            this.joinKey = joinKey;
+        }
+    }
 }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java
index 50bca16aa..e89259073 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java
@@ -18,90 +18,18 @@
 
 package org.apache.paimon.flink.lookup;
 
-import org.apache.paimon.CoreOptions;
 import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.flink.FlinkConnectorOptions.LookupCacheMode;
-import org.apache.paimon.lookup.RocksDBStateFactory;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.Table;
-import org.apache.paimon.types.RowType;
 
-import java.io.File;
+import java.io.Closeable;
 import java.io.IOException;
-import java.util.HashSet;
 import java.util.List;
-import java.util.function.Predicate;
-
-import static org.apache.paimon.CoreOptions.MergeEngine.DEDUPLICATE;
-import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_CACHE_MODE;
 
 /** A lookup table which provides get and refresh. */
-public interface LookupTable {
-
-    List<InternalRow> get(InternalRow key) throws IOException;
-
-    static FullCacheLookupTable create(
-            RocksDBStateFactory stateFactory,
-            RowType rowType,
-            List<String> primaryKey,
-            List<String> joinKey,
-            Predicate<InternalRow> recordFilter,
-            long lruCacheSize)
-            throws IOException {
-        if (primaryKey.isEmpty()) {
-            return new NoPrimaryKeyLookupTable(
-                    stateFactory, rowType, joinKey, recordFilter, lruCacheSize);
-        } else {
-            if (new HashSet<>(primaryKey).equals(new HashSet<>(joinKey))) {
-                return new PrimaryKeyLookupTable(
-                        stateFactory, rowType, joinKey, recordFilter, lruCacheSize);
-            } else {
-                return new SecondaryIndexLookupTable(
-                        stateFactory, rowType, primaryKey, joinKey, recordFilter, lruCacheSize);
-            }
-        }
-    }
+public interface LookupTable extends Closeable {
 
-    static PartialCacheLookupTable create(
-            Table table, int[] projection, List<String> joinKey, File tempPath) {
-        validatePartialCacheMode(table, joinKey);
-        return new PrimaryKeyPartialLookupTable((FileStoreTable) table, projection, tempPath);
-    }
+    void open() throws Exception;
 
-    static void validatePartialCacheMode(Table table, List<String> joinKey) {
-        CoreOptions options = new CoreOptions(table.options());
-        if (options.toConfiguration().get(LOOKUP_CACHE_MODE) != LookupCacheMode.PARTIAL) {
-            return;
-        }
-        List<String> primaryKey = table.primaryKeys();
-        if (!new HashSet<>(primaryKey).equals(new HashSet<>(joinKey))) {
-            throw new UnsupportedOperationException(
-                    String.format(
-                            "For partial cache mode, the join key: %s must equal to the primary key: %s",
-                            joinKey, primaryKey));
-        }
-
-        if (table.partitionKeys().size() > 0) {
-            throw new UnsupportedOperationException(
-                    "The partitioned table are not supported in partial cache mode.");
-        }
-
-        if (new CoreOptions(table.options()).bucket() <= 0) {
-            throw new UnsupportedOperationException(
-                    "Only support fixed bucket mode in partial cache mode.");
-        }
-
-        if (options.changelogProducer() != CoreOptions.ChangelogProducer.LOOKUP) {
-            if (options.sequenceField().isPresent()) {
-                throw new UnsupportedOperationException(
-                        "Not support sequence field definition, but is: "
-                                + options.sequenceField().get());
-            }
+    List<InternalRow> get(InternalRow key) throws IOException;
 
-            if (options.mergeEngine() != DEDUPLICATE) {
-                throw new UnsupportedOperationException(
-                        "Only support deduplicate merge engine, but is: " + options.mergeEngine());
-            }
-        }
-    }
+    void refresh() throws Exception;
 }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/NoPrimaryKeyLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/NoPrimaryKeyLookupTable.java
index d6979e9b7..329cd0d6a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/NoPrimaryKeyLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/NoPrimaryKeyLookupTable.java
@@ -24,9 +24,7 @@ import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.data.serializer.InternalSerializers;
 import org.apache.paimon.lookup.BulkLoader;
 import org.apache.paimon.lookup.RocksDBListState;
-import org.apache.paimon.lookup.RocksDBStateFactory;
 import org.apache.paimon.types.RowKind;
-import org.apache.paimon.types.RowType;
 import org.apache.paimon.utils.KeyProjectedRow;
 import org.apache.paimon.utils.TypeUtils;
 
@@ -35,38 +33,30 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
-import java.util.function.Predicate;
 
 /** A {@link LookupTable} for table without primary key. */
-public class NoPrimaryKeyLookupTable implements FullCacheLookupTable {
+public class NoPrimaryKeyLookupTable extends FullCacheLookupTable {
 
     private final RocksDBListState<InternalRow, InternalRow> state;
 
-    private final Predicate<InternalRow> recordFilter;
-
     private final KeyProjectedRow joinKeyRow;
 
-    public NoPrimaryKeyLookupTable(
-            RocksDBStateFactory stateFactory,
-            RowType rowType,
-            List<String> joinKeys,
-            Predicate<InternalRow> recordFilter,
-            long lruCacheSize)
-            throws IOException {
-        List<String> fieldNames = rowType.getFieldNames();
-        int[] joinKeyMapping = joinKeys.stream().mapToInt(fieldNames::indexOf).toArray();
+    public NoPrimaryKeyLookupTable(Context context, long lruCacheSize) throws IOException {
+        super(context);
+        List<String> fieldNames = projectedType.getFieldNames();
+        int[] joinKeyMapping = context.joinKey.stream().mapToInt(fieldNames::indexOf).toArray();
         this.joinKeyRow = new KeyProjectedRow(joinKeyMapping);
         this.state =
                 stateFactory.listState(
                         "join-key-index",
-                        InternalSerializers.create(TypeUtils.project(rowType, joinKeyMapping)),
-                        InternalSerializers.create(rowType),
+                        InternalSerializers.create(
+                                TypeUtils.project(projectedType, joinKeyMapping)),
+                        InternalSerializers.create(projectedType),
                         lruCacheSize);
-        this.recordFilter = recordFilter;
     }
 
     @Override
-    public List<InternalRow> get(InternalRow key) throws IOException {
+    public List<InternalRow> innerGet(InternalRow key) throws IOException {
         return state.get(key);
     }
 
@@ -82,7 +72,7 @@ public class NoPrimaryKeyLookupTable implements FullCacheLookupTable {
             InternalRow row = incremental.next();
             joinKeyRow.replaceRow(row);
             if (row.getRowKind() == RowKind.INSERT || row.getRowKind() == RowKind.UPDATE_AFTER) {
-                if (recordFilter.test(row)) {
+                if (recordFilter().test(row)) {
                     state.add(joinKeyRow, row);
                 }
             } else {
@@ -94,11 +84,6 @@ public class NoPrimaryKeyLookupTable implements FullCacheLookupTable {
         }
     }
 
-    @Override
-    public Predicate<InternalRow> recordFilter() {
-        return recordFilter;
-    }
-
     @Override
     public byte[] toKeyBytes(InternalRow row) throws IOException {
         joinKeyRow.replaceRow(row);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartialCacheLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartialCacheLookupTable.java
deleted file mode 100644
index a82507ffa..000000000
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PartialCacheLookupTable.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- *
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- */
-
-package org.apache.paimon.flink.lookup;
-
-import org.apache.paimon.table.source.Split;
-
-import java.util.List;
-
-/** Lookup table of partial cache. */
-public interface PartialCacheLookupTable extends LookupTable {
-    void refresh(List<Split> splits);
-}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyLookupTable.java
index d8707a4ab..9ef35bea8 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyLookupTable.java
@@ -21,10 +21,8 @@ package org.apache.paimon.flink.lookup;
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.data.serializer.InternalSerializers;
 import org.apache.paimon.lookup.BulkLoader;
-import org.apache.paimon.lookup.RocksDBStateFactory;
 import org.apache.paimon.lookup.RocksDBValueState;
 import org.apache.paimon.types.RowKind;
-import org.apache.paimon.types.RowType;
 import org.apache.paimon.utils.KeyProjectedRow;
 import org.apache.paimon.utils.TypeUtils;
 
@@ -32,44 +30,34 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.function.Predicate;
 
 /** A {@link LookupTable} for primary key table. */
-public class PrimaryKeyLookupTable implements FullCacheLookupTable {
+public class PrimaryKeyLookupTable extends FullCacheLookupTable {
 
     protected final RocksDBValueState<InternalRow, InternalRow> tableState;
 
-    protected final RowType rowType;
-
-    protected final Predicate<InternalRow> recordFilter;
-
     protected int[] primaryKeyMapping;
 
-    protected final KeyProjectedRow primaryKey;
+    protected final KeyProjectedRow primaryKeyRow;
 
-    public PrimaryKeyLookupTable(
-            RocksDBStateFactory stateFactory,
-            RowType rowType,
-            List<String> primaryKey,
-            Predicate<InternalRow> recordFilter,
-            long lruCacheSize)
-            throws IOException {
-        this.rowType = rowType;
-        List<String> fieldNames = rowType.getFieldNames();
-        this.primaryKeyMapping = primaryKey.stream().mapToInt(fieldNames::indexOf).toArray();
-        this.primaryKey = new KeyProjectedRow(primaryKeyMapping);
+    public PrimaryKeyLookupTable(Context context, long lruCacheSize) throws IOException {
+        super(context);
+        List<String> fieldNames = projectedType.getFieldNames();
+        this.primaryKeyMapping =
+                context.table.primaryKeys().stream().mapToInt(fieldNames::indexOf).toArray();
+        this.primaryKeyRow = new KeyProjectedRow(primaryKeyMapping);
 
         this.tableState =
                 stateFactory.valueState(
                         "table",
-                        InternalSerializers.create(TypeUtils.project(rowType, primaryKeyMapping)),
-                        InternalSerializers.create(rowType),
+                        InternalSerializers.create(
+                                TypeUtils.project(projectedType, primaryKeyMapping)),
+                        InternalSerializers.create(projectedType),
                         lruCacheSize);
-        this.recordFilter = recordFilter;
     }
 
     @Override
-    public List<InternalRow> get(InternalRow key) throws IOException {
+    public List<InternalRow> innerGet(InternalRow key) throws IOException {
         InternalRow value = tableState.get(key);
         return value == null ? Collections.emptyList() : Collections.singletonList(value);
     }
@@ -79,38 +67,33 @@ public class PrimaryKeyLookupTable implements FullCacheLookupTable {
             throws IOException {
         while (incremental.hasNext()) {
             InternalRow row = incremental.next();
-            primaryKey.replaceRow(row);
+            primaryKeyRow.replaceRow(row);
             if (orderByLastField) {
-                InternalRow previous = tableState.get(primaryKey);
-                int orderIndex = rowType.getFieldCount() - 1;
+                InternalRow previous = tableState.get(primaryKeyRow);
+                int orderIndex = projectedType.getFieldCount() - 1;
                 if (previous != null && previous.getLong(orderIndex) > row.getLong(orderIndex)) {
                     continue;
                 }
             }
 
             if (row.getRowKind() == RowKind.INSERT || row.getRowKind() == RowKind.UPDATE_AFTER) {
-                if (recordFilter.test(row)) {
-                    tableState.put(primaryKey, row);
+                if (recordFilter().test(row)) {
+                    tableState.put(primaryKeyRow, row);
                 } else {
                     // The new record under primary key is filtered
                     // We need to delete this primary key as it no longer exists.
-                    tableState.delete(primaryKey);
+                    tableState.delete(primaryKeyRow);
                 }
             } else {
-                tableState.delete(primaryKey);
+                tableState.delete(primaryKeyRow);
             }
         }
     }
 
-    @Override
-    public Predicate<InternalRow> recordFilter() {
-        return recordFilter;
-    }
-
     @Override
     public byte[] toKeyBytes(InternalRow row) throws IOException {
-        primaryKey.replaceRow(row);
-        return tableState.serializeKey(primaryKey);
+        primaryKeyRow.replaceRow(row);
+        return tableState.serializeKey(primaryKeyRow);
     }
 
     @Override
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java
index 66aa965b4..57a274e11 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java
@@ -22,14 +22,15 @@ import org.apache.paimon.data.BinaryRow;
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.disk.IOManagerImpl;
 import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.table.BucketMode;
 import org.apache.paimon.table.FileStoreTable;
 import org.apache.paimon.table.query.TableQuery;
 import org.apache.paimon.table.source.DataSplit;
 import org.apache.paimon.table.source.Split;
 import org.apache.paimon.utils.Projection;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
 
 import java.io.File;
 import java.io.IOException;
@@ -37,20 +38,37 @@ import java.util.Collections;
 import java.util.List;
 
 /** Lookup table for primary key which supports to read the LSM tree directly. */
-public class PrimaryKeyPartialLookupTable implements PartialCacheLookupTable {
+public class PrimaryKeyPartialLookupTable implements LookupTable {
 
-    private static final Logger LOG = LoggerFactory.getLogger(PrimaryKeyPartialLookupTable.class);
-
-    private final FixedBucketKeyExtractor extractor;
+    private final FixedBucketFromPkExtractor extractor;
 
     private final TableQuery tableQuery;
 
-    public PrimaryKeyPartialLookupTable(FileStoreTable table, int[] projection, File tempPath) {
+    private final TableFileMonitor fileMonitor;
+
+    public PrimaryKeyPartialLookupTable(
+            FileStoreTable table, @Nullable Predicate predicate, int[] projection, File tempPath) {
+        if (table.partitionKeys().size() > 0) {
+            throw new UnsupportedOperationException(
+                    "The partitioned table are not supported in partial cache mode.");
+        }
+
+        if (table.bucketMode() != BucketMode.FIXED) {
+            throw new UnsupportedOperationException(
+                    "Unsupported mode for partial lookup: " + table.bucketMode());
+        }
+
         this.tableQuery =
                 table.newTableQuery()
                         .withValueProjection(Projection.of(projection).toNestedIndexes())
                         .withIOManager(new IOManagerImpl(tempPath.toString()));
-        this.extractor = new FixedBucketKeyExtractor(table.schema());
+        this.extractor = new FixedBucketFromPkExtractor(table.schema());
+        this.fileMonitor = new TableFileMonitor(table, predicate);
+    }
+
+    @Override
+    public void open() throws Exception {
+        refresh();
     }
 
     @Override
@@ -68,7 +86,8 @@ public class PrimaryKeyPartialLookupTable implements PartialCacheLookupTable {
     }
 
     @Override
-    public void refresh(List<Split> splits) {
+    public void refresh() {
+        List<Split> splits = fileMonitor.readChanges();
         for (Split split : splits) {
             if (!(split instanceof DataSplit)) {
                 throw new IllegalArgumentException("Unsupported split: " + split.getClass());
@@ -81,4 +100,9 @@ public class PrimaryKeyPartialLookupTable implements PartialCacheLookupTable {
             tableQuery.refreshFiles(partition, bucket, before, after);
         }
     }
+
+    @Override
+    public void close() throws IOException {
+        tableQuery.close();
+    }
 }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/SecondaryIndexLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/SecondaryIndexLookupTable.java
index 8de3fa34d..793b99c5b 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/SecondaryIndexLookupTable.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/SecondaryIndexLookupTable.java
@@ -21,9 +21,7 @@ package org.apache.paimon.flink.lookup;
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.data.serializer.InternalSerializers;
 import org.apache.paimon.lookup.RocksDBSetState;
-import org.apache.paimon.lookup.RocksDBStateFactory;
 import org.apache.paimon.types.RowKind;
-import org.apache.paimon.types.RowType;
 import org.apache.paimon.utils.KeyProjectedRow;
 import org.apache.paimon.utils.TypeUtils;
 
@@ -31,7 +29,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.function.Predicate;
 
 /** A {@link LookupTable} for primary key table which provides lookup by secondary key. */
 public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
@@ -40,28 +37,22 @@ public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
 
     private final KeyProjectedRow secKeyRow;
 
-    public SecondaryIndexLookupTable(
-            RocksDBStateFactory stateFactory,
-            RowType rowType,
-            List<String> primaryKey,
-            List<String> secKey,
-            Predicate<InternalRow> recordFilter,
-            long lruCacheSize)
-            throws IOException {
-        super(stateFactory, rowType, primaryKey, recordFilter, lruCacheSize / 2);
-        List<String> fieldNames = rowType.getFieldNames();
-        int[] secKeyMapping = secKey.stream().mapToInt(fieldNames::indexOf).toArray();
+    public SecondaryIndexLookupTable(Context context, long lruCacheSize) throws IOException {
+        super(context, lruCacheSize / 2);
+        List<String> fieldNames = projectedType.getFieldNames();
+        int[] secKeyMapping = context.joinKey.stream().mapToInt(fieldNames::indexOf).toArray();
         this.secKeyRow = new KeyProjectedRow(secKeyMapping);
         this.indexState =
                 stateFactory.setState(
                         "sec-index",
-                        InternalSerializers.create(TypeUtils.project(rowType, secKeyMapping)),
-                        InternalSerializers.create(TypeUtils.project(rowType, primaryKeyMapping)),
+                        InternalSerializers.create(TypeUtils.project(projectedType, secKeyMapping)),
+                        InternalSerializers.create(
+                                TypeUtils.project(projectedType, primaryKeyMapping)),
                         lruCacheSize / 2);
     }
 
     @Override
-    public List<InternalRow> get(InternalRow key) throws IOException {
+    public List<InternalRow> innerGet(InternalRow key) throws IOException {
         List<InternalRow> pks = indexState.get(key);
         List<InternalRow> values = new ArrayList<>(pks.size());
         for (InternalRow pk : pks) {
@@ -78,14 +69,14 @@ public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
             throws IOException {
         while (incremental.hasNext()) {
             InternalRow row = incremental.next();
-            primaryKey.replaceRow(row);
+            primaryKeyRow.replaceRow(row);
 
             boolean previousFetched = false;
             InternalRow previous = null;
             if (orderByLastField) {
-                previous = tableState.get(primaryKey);
+                previous = tableState.get(primaryKeyRow);
                 previousFetched = true;
-                int orderIndex = rowType.getFieldCount() - 1;
+                int orderIndex = projectedType.getFieldCount() - 1;
                 if (previous != null && previous.getLong(orderIndex) > row.getLong(orderIndex)) {
                     continue;
                 }
@@ -93,21 +84,21 @@ public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
 
             if (row.getRowKind() == RowKind.INSERT || row.getRowKind() == RowKind.UPDATE_AFTER) {
                 if (!previousFetched) {
-                    previous = tableState.get(primaryKey);
+                    previous = tableState.get(primaryKeyRow);
                 }
                 if (previous != null) {
-                    indexState.retract(secKeyRow.replaceRow(previous), primaryKey);
+                    indexState.retract(secKeyRow.replaceRow(previous), primaryKeyRow);
                 }
 
-                if (recordFilter.test(row)) {
-                    tableState.put(primaryKey, row);
-                    indexState.add(secKeyRow.replaceRow(row), primaryKey);
+                if (recordFilter().test(row)) {
+                    tableState.put(primaryKeyRow, row);
+                    indexState.add(secKeyRow.replaceRow(row), primaryKeyRow);
                 } else {
-                    tableState.delete(primaryKey);
+                    tableState.delete(primaryKeyRow);
                 }
             } else {
-                tableState.delete(primaryKey);
-                indexState.retract(secKeyRow.replaceRow(row), primaryKey);
+                tableState.delete(primaryKeyRow);
+                indexState.retract(secKeyRow.replaceRow(row), primaryKeyRow);
             }
         }
     }
@@ -115,6 +106,6 @@ public class SecondaryIndexLookupTable extends PrimaryKeyLookupTable {
     @Override
     public void bulkLoadWritePlus(byte[] key, byte[] value) throws IOException {
         InternalRow row = tableState.deserializeValue(value);
-        indexState.add(secKeyRow.replaceRow(row), primaryKey.replaceRow(row));
+        indexState.add(secKeyRow.replaceRow(row), primaryKeyRow.replaceRow(row));
     }
 }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java
index 81e13a857..208691477 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java
@@ -29,7 +29,6 @@ import org.apache.paimon.flink.log.LogSourceProvider;
 import org.apache.paimon.flink.log.LogStoreTableFactory;
 import org.apache.paimon.flink.lookup.FileStoreLookupFunction;
 import org.apache.paimon.flink.lookup.LookupRuntimeProviderFactory;
-import org.apache.paimon.flink.lookup.LookupTable;
 import org.apache.paimon.options.Options;
 import org.apache.paimon.predicate.Predicate;
 import org.apache.paimon.table.AppendOnlyFileStoreTable;
@@ -55,10 +54,8 @@ import org.apache.flink.table.plan.stats.TableStats;
 import javax.annotation.Nullable;
 
 import java.time.Duration;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static org.apache.paimon.CoreOptions.CHANGELOG_PRODUCER;
@@ -305,11 +302,6 @@ public class DataTableSource extends FlinkTableSource {
         Options options = new Options(table.options());
         boolean enableAsync = options.get(LOOKUP_ASYNC);
         int asyncThreadNumber = options.get(LOOKUP_ASYNC_THREAD_NUMBER);
-        LookupTable.validatePartialCacheMode(
-                table,
-                Arrays.stream(joinKey)
-                        .mapToObj(i -> table.rowType().getFieldNames().get(projection[i]))
-                        .collect(Collectors.toList()));
         return LookupRuntimeProviderFactory.create(
                 new FileStoreLookupFunction(table, projection, joinKey, predicate),
                 enableAsync,
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
index 711f58633..76b4fd1c5 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java
@@ -53,15 +53,17 @@ public class LookupJoinITCase extends CatalogITCaseBase {
                 "CREATE TABLE PARTITIONED_DIM (i INT, j INT, k1 INT, k2 INT, PRIMARY KEY (i, j) NOT ENFORCED)"
                         + "PARTITIONED BY (`i`) WITH ('continuous.discovery-interval'='1 ms' %s)";
 
-        String lruOption =
-                ", 'changelog-producer'='lookup', 'lookup.cache' = 'partial', 'bucket' = '2'";
+        String fullOption =
+                ", 'changelog-producer'='lookup', 'lookup.cache' = 'full', 'bucket' = '2'";
+
+        String lruOption = ", 'changelog-producer'='lookup', 'bucket' = '2'";
 
         switch (cacheMode) {
             case FULL:
-                tEnv.executeSql(String.format(dim, ""));
-                tEnv.executeSql(String.format(partitioned, ""));
+                tEnv.executeSql(String.format(dim, fullOption));
+                tEnv.executeSql(String.format(partitioned, fullOption));
                 break;
-            case PARTIAL:
+            case AUTO:
                 tEnv.executeSql(String.format(dim, lruOption));
                 tEnv.executeSql(String.format(partitioned, lruOption));
                 break;
@@ -313,23 +315,14 @@ public class LookupJoinITCase extends CatalogITCaseBase {
         iterator.close();
     }
 
-    @ParameterizedTest
-    @EnumSource(LookupCacheMode.class)
-    public void testNonPkLookup(LookupCacheMode cacheMode) throws Exception {
-        initTable(cacheMode);
+    @Test
+    public void testNonPkLookup() throws Exception {
+        initTable(LookupCacheMode.AUTO);
         sql("INSERT INTO DIM VALUES (1, 11, 111, 1111), (2, 22, 222, 2222), (3, 22, 333, 3333)");
 
         String query =
                 "SELECT D.i, T.i, D.k1, D.k2 FROM T LEFT JOIN DIM for system_time as of T.proctime AS D ON T.i = D.j";
-        BlockingIterator<Row, Row> iterator;
-        if (cacheMode == LookupCacheMode.PARTIAL) {
-            assertThatThrownBy(() -> sEnv.executeSql(query))
-                    .hasRootCauseMessage(
-                            "For partial cache mode, the join key: [j] must equal to the primary key: [i]");
-            return;
-        } else {
-            iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
-        }
+        BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
 
         sql("INSERT INTO T VALUES (11), (22), (33)");
         List<Row> result = iterator.collect(4);
@@ -602,21 +595,12 @@ public class LookupJoinITCase extends CatalogITCaseBase {
         iterator.close();
     }
 
-    @ParameterizedTest
-    @EnumSource(LookupCacheMode.class)
-    public void testLookupPartitionedTable(LookupCacheMode cacheMode) throws Exception {
-        initTable(cacheMode);
+    @Test
+    public void testLookupPartitionedTable() throws Exception {
+        initTable(LookupCacheMode.AUTO);
         String query =
                 "SELECT T.i, D.j, D.k1, D.k2 FROM T LEFT JOIN PARTITIONED_DIM for system_time as of T.proctime AS D ON T.i = D.i";
-        BlockingIterator<Row, Row> iterator;
-        if (cacheMode == LookupCacheMode.PARTIAL) {
-            assertThatThrownBy(() -> sEnv.executeSql(query))
-                    .hasRootCauseMessage(
-                            "For partial cache mode, the join key: [i] must equal to the primary key: [i, j]");
-            return;
-        } else {
-            iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
-        }
+        BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
 
         sql("INSERT INTO T VALUES (1), (2), (3)");
 
@@ -639,26 +623,16 @@ public class LookupJoinITCase extends CatalogITCaseBase {
         iterator.close();
     }
 
-    @ParameterizedTest
-    @EnumSource(LookupCacheMode.class)
-    public void testLookupNonPkAppendTable(LookupCacheMode cacheMode) throws Exception {
+    @Test
+    public void testLookupNonPkAppendTable() throws Exception {
         sql(
                 "CREATE TABLE DIM_NO_PK (i INT, j INT, k1 INT, k2 INT) "
-                        + "PARTITIONED BY (`i`) WITH ('continuous.discovery-interval'='1 ms' %s)",
-                cacheMode == LookupCacheMode.PARTIAL ? ", 'lookup.cache' = 'partial'" : "");
+                        + "PARTITIONED BY (`i`) WITH ('continuous.discovery-interval'='1 ms')");
 
         String query =
                 "SELECT T.i, D.j, D.k1, D.k2 FROM T LEFT JOIN DIM_NO_PK for system_time as of T.proctime AS D ON T.i "
                         + "= D.i";
-        BlockingIterator<Row, Row> iterator;
-        if (cacheMode == LookupCacheMode.PARTIAL) {
-            assertThatThrownBy(() -> sEnv.executeSql(query))
-                    .hasRootCauseMessage(
-                            "For partial cache mode, the join key: [i] must equal to the primary key: []");
-            return;
-        } else {
-            iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
-        }
+        BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
 
         sql("INSERT INTO T VALUES (1), (2), (3)");
 
@@ -771,27 +745,4 @@ public class LookupJoinITCase extends CatalogITCaseBase {
 
         iterator.close();
     }
-
-    @Test
-    public void testPartialCacheValidation() {
-        sql(
-                "CREATE TABLE DIM2 (i INT PRIMARY KEY NOT ENFORCED, j INT, k1 INT, k2 INT) WITH"
-                        + " ('continuous.discovery-interval'='1 ms', 'changelog-producer' = 'lookup', "
-                        + "'lookup.cache' = 'partial', 'bucket' = '-1')");
-
-        String query =
-                "SELECT T.i, D.k1 FROM T LEFT JOIN DIM2 for system_time as of T.proctime AS D ON T.i = D.i";
-        assertThatThrownBy(() -> sEnv.executeSql(query))
-                .hasRootCauseMessage("Only support fixed bucket mode in partial cache mode.");
-        sql(
-                "CREATE TABLE DIM3 (i INT PRIMARY KEY NOT ENFORCED, j INT, k1 INT, k2 INT) WITH"
-                        + " ('continuous.discovery-interval'='1 ms', "
-                        + "'lookup.cache' = 'partial', 'bucket' = '1', 'merge-engine' = 'aggregation')");
-
-        String query2 =
-                "SELECT T.i, D.k1 FROM T LEFT JOIN DIM3 for system_time as of T.proctime AS D ON T.i = D.i";
-
-        assertThatThrownBy(() -> sEnv.executeSql(query2))
-                .hasRootCauseMessage("Only support deduplicate merge engine, but is: aggregation");
-    }
 }
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java
index 5741cbb7c..e67e5eb76 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java
@@ -19,6 +19,7 @@
 package org.apache.paimon.flink.lookup;
 
 import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Identifier;
 import org.apache.paimon.data.GenericRow;
 import org.apache.paimon.data.InternalRow;
 import org.apache.paimon.data.JoinedRow;
@@ -27,8 +28,6 @@ import org.apache.paimon.disk.IOManagerImpl;
 import org.apache.paimon.flink.lookup.FullCacheLookupTable.TableBulkLoader;
 import org.apache.paimon.fs.FileIO;
 import org.apache.paimon.fs.local.LocalFileIO;
-import org.apache.paimon.lookup.BulkLoader;
-import org.apache.paimon.lookup.RocksDBStateFactory;
 import org.apache.paimon.options.Options;
 import org.apache.paimon.schema.Schema;
 import org.apache.paimon.schema.SchemaManager;
@@ -61,8 +60,8 @@ import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ThreadLocalRandom;
 
+import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
-import static org.apache.paimon.schema.SystemColumns.SEQUENCE_NUMBER;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
@@ -71,36 +70,50 @@ public class LookupTableTest extends TableTestBase {
 
     @TempDir Path tempDir;
 
-    private RocksDBStateFactory stateFactory;
-
     private RowType rowType;
 
     private IOManager ioManager;
 
+    private FullCacheLookupTable table;
+
     @BeforeEach
     public void before() throws IOException {
-        this.stateFactory = new RocksDBStateFactory(tempDir.toString(), new Options(), null);
         this.rowType = RowType.of(new IntType(), new IntType(), new IntType());
         this.ioManager = new IOManagerImpl(tempDir.toString());
     }
 
     @AfterEach
     public void after() throws IOException {
-        if (stateFactory != null) {
-            stateFactory.close();
+        if (table != null) {
+            table.close();
         }
     }
 
+    private FileStoreTable createTable(List<String> primaryKeys, Options options) throws Exception {
+        Identifier identifier = new Identifier("default", "t");
+        Schema schema =
+                new Schema(
+                        rowType.getFields(),
+                        Collections.emptyList(),
+                        primaryKeys,
+                        options.toMap(),
+                        null);
+        catalog.createTable(identifier, schema, false);
+        return (FileStoreTable) catalog.getTable(identifier);
+    }
+
     @Test
-    public void testPkTable() throws IOException, BulkLoader.WriteException {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType,
-                        singletonList("f0"),
-                        singletonList("f0"),
+    public void testPkTable() throws Exception {
+        FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> true,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f0"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         // test bulk load error
         {
@@ -147,14 +160,18 @@ public class LookupTableTest extends TableTestBase {
 
     @Test
     public void testPkTableWithSequenceField() throws Exception {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType.appendDataField(SEQUENCE_NUMBER, DataTypes.BIGINT()),
-                        singletonList("f0"),
-                        singletonList("f0"),
+        Options options = new Options();
+        options.set(CoreOptions.SEQUENCE_FIELD, "f1");
+        FileStoreTable storeTable = createTable(singletonList("f0"), options);
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> true,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f0"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         List<Pair<byte[], byte[]>> records = new ArrayList<>();
         for (int i = 1; i <= 10; i++) {
@@ -172,31 +189,33 @@ public class LookupTableTest extends TableTestBase {
         table.refresh(singletonList(sequence(row(1, 22, 222), 1L)).iterator(), true);
         List<InternalRow> result = table.get(row(1));
         assertThat(result).hasSize(1);
-        assertRow(result.get(0), 1, 22, 222, 1);
+        assertRow(result.get(0), 1, 22, 222);
 
         // refresh with old sequence
         table.refresh(singletonList((sequence(row(1, 33, 333), 0L))).iterator(), true);
         result = table.get(row(1));
         assertThat(result).hasSize(1);
-        assertRow(result.get(0), 1, 22, 222, 1);
+        assertRow(result.get(0), 1, 22, 222);
 
         // test refresh delete data with old sequence
         table.refresh(
                 singletonList(sequence(row(RowKind.DELETE, 1, 11, 111), -1L)).iterator(), true);
         assertThat(table.get(row(1))).hasSize(1);
-        assertRow(result.get(0), 1, 22, 222, 1);
+        assertRow(result.get(0), 1, 22, 222);
     }
 
     @Test
-    public void testPkTablePkFilter() throws IOException {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType,
-                        singletonList("f0"),
-                        singletonList("f0"),
+    public void testPkTablePkFilter() throws Exception {
+        FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> r.getInt(0) < 3,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f0"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         table.refresh(singletonList(sequence(row(1, 11, 111), -1L)).iterator(), false);
         List<InternalRow> result = table.get(row(1));
@@ -217,15 +236,17 @@ public class LookupTableTest extends TableTestBase {
     }
 
     @Test
-    public void testPkTableNonPkFilter() throws IOException {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType,
-                        singletonList("f0"),
-                        singletonList("f0"),
+    public void testPkTableNonPkFilter() throws Exception {
+        FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> r.getInt(1) < 22,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f0"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         table.refresh(singletonList(sequence(row(1, 11, 111), -1L)).iterator(), false);
         List<InternalRow> result = table.get(row(1));
@@ -238,15 +259,17 @@ public class LookupTableTest extends TableTestBase {
     }
 
     @Test
-    public void testSecKeyTable() throws IOException, BulkLoader.WriteException {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType,
-                        singletonList("f0"),
-                        singletonList("f1"),
+    public void testSecKeyTable() throws Exception {
+        FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> true,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f1"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         // test bulk load 100_000 records
         List<Pair<byte[], byte[]>> records = new ArrayList<>();
@@ -279,14 +302,18 @@ public class LookupTableTest extends TableTestBase {
 
     @Test
     public void testSecKeyTableWithSequenceField() throws Exception {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType.appendDataField(SEQUENCE_NUMBER, DataTypes.BIGINT()),
-                        singletonList("f0"),
-                        singletonList("f1"),
+        Options options = new Options();
+        options.set(CoreOptions.SEQUENCE_FIELD, "f1");
+        FileStoreTable storeTable = createTable(singletonList("f0"), options);
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> true,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f1"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         List<Pair<byte[], byte[]>> records = new ArrayList<>();
         Random rnd = new Random();
@@ -318,7 +345,7 @@ public class LookupTableTest extends TableTestBase {
                 true);
         List<InternalRow> result = table.get(row(22));
         assertThat(result.stream().map(row -> row.getInt(0))).contains(1);
-        assertThat(result.stream().map(InternalRow::getFieldCount)).allMatch(n -> n == 4);
+        assertThat(result.stream().map(InternalRow::getFieldCount)).allMatch(n -> n == 3);
 
         // refresh with old value
         table.refresh(
@@ -330,15 +357,17 @@ public class LookupTableTest extends TableTestBase {
     }
 
     @Test
-    public void testSecKeyTablePkFilter() throws IOException {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType,
-                        singletonList("f0"),
-                        singletonList("f1"),
+    public void testSecKeyTablePkFilter() throws Exception {
+        FileStoreTable storeTable = createTable(singletonList("f0"), new Options());
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> r.getInt(0) < 3,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f1"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         table.refresh(singletonList(sequence(row(1, 11, 111), -1L)).iterator(), false);
         List<InternalRow> result = table.get(row(11));
@@ -368,15 +397,17 @@ public class LookupTableTest extends TableTestBase {
     }
 
     @Test
-    public void testNoPrimaryKeyTable() throws IOException, BulkLoader.WriteException {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType,
-                        Collections.emptyList(),
-                        singletonList("f1"),
+    public void testNoPrimaryKeyTable() throws Exception {
+        FileStoreTable storeTable = createTable(emptyList(), new Options());
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> true,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f1"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         // test bulk load 100_000 records
         List<Pair<byte[], byte[]>> records = new ArrayList<>();
@@ -408,15 +439,17 @@ public class LookupTableTest extends TableTestBase {
     }
 
     @Test
-    public void testNoPrimaryKeyTableFilter() throws IOException {
-        FullCacheLookupTable table =
-                LookupTable.create(
-                        stateFactory,
-                        rowType,
-                        Collections.emptyList(),
-                        singletonList("f1"),
+    public void testNoPrimaryKeyTableFilter() throws Exception {
+        FileStoreTable storeTable = createTable(emptyList(), new Options());
+        FullCacheLookupTable.Context context =
+                new FullCacheLookupTable.Context(
+                        storeTable,
+                        new int[] {0, 1, 2},
+                        null,
+                        tempDir.toFile(),
                         r -> r.getInt(2) < 222,
-                        ThreadLocalRandom.current().nextInt(2) * 10);
+                        singletonList("f1"));
+        table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10);
 
         table.refresh(singletonList(row(1, 11, 333)).iterator(), false);
         List<InternalRow> result = table.get(row(11));
@@ -437,13 +470,9 @@ public class LookupTableTest extends TableTestBase {
     @Test
     public void testPartialLookupTable() throws Exception {
         FileStoreTable dimTable = createDimTable();
-        PartialCacheLookupTable table =
-                LookupTable.create(
-                        dimTable,
-                        new int[] {0, 1, 2},
-                        Collections.singletonList("pk"),
-                        tempDir.toFile());
-        TableFileMonitor monitor = new TableFileMonitor(dimTable, null);
+        PrimaryKeyPartialLookupTable table =
+                new PrimaryKeyPartialLookupTable(
+                        dimTable, null, new int[] {0, 1, 2}, tempDir.toFile());
         List<InternalRow> result = table.get(row(1, -1));
         assertThat(result).hasSize(0);
 
@@ -451,7 +480,7 @@ public class LookupTableTest extends TableTestBase {
         result = table.get(row(1, -1));
         assertThat(result).hasSize(0);
 
-        table.refresh(monitor.readChanges());
+        table.refresh();
         result = table.get(row(1, -1));
         assertThat(result).hasSize(1);
         assertRow(result.get(0), 1, -1, 11);
@@ -460,7 +489,7 @@ public class LookupTableTest extends TableTestBase {
         assertRow(result.get(0), 2, -2, 22);
 
         write(dimTable, ioManager, GenericRow.ofKind(RowKind.DELETE, 1, -1, 11));
-        table.refresh(monitor.readChanges());
+        table.refresh();
         result = table.get(row(1, -1));
         assertThat(result).hasSize(0);
     }
@@ -468,13 +497,9 @@ public class LookupTableTest extends TableTestBase {
     @Test
     public void testPartialLookupTableWithProjection() throws Exception {
         FileStoreTable dimTable = createDimTable();
-        PartialCacheLookupTable table =
-                LookupTable.create(
-                        dimTable,
-                        new int[] {2, 1},
-                        Collections.singletonList("pk"),
-                        tempDir.toFile());
-        TableFileMonitor monitor = new TableFileMonitor(dimTable, null);
+        PrimaryKeyPartialLookupTable table =
+                new PrimaryKeyPartialLookupTable(
+                        dimTable, null, new int[] {2, 1}, tempDir.toFile());
         List<InternalRow> result = table.get(row(1, -1));
         assertThat(result).hasSize(0);
 
@@ -482,7 +507,7 @@ public class LookupTableTest extends TableTestBase {
         result = table.get(row(1, -1));
         assertThat(result).hasSize(0);
 
-        table.refresh(monitor.readChanges());
+        table.refresh();
         result = table.get(row(1, -1));
         assertThat(result).hasSize(1);
         assertRow(result.get(0), 11, -1);