You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2023/02/09 05:36:12 UTC

[flink-table-store] branch master updated: [FLINK-30918] Refactor CompositePkAndMultiPartitionedTableITCase to get rid of managed table

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git


The following commit(s) were added to refs/heads/master by this push:
     new ac01ddc3 [FLINK-30918] Refactor CompositePkAndMultiPartitionedTableITCase to get rid of managed table
ac01ddc3 is described below

commit ac01ddc333372dc92bf9bc3080b7dd4d90d26d01
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Thu Feb 9 13:36:07 2023 +0800

    [FLINK-30918] Refactor CompositePkAndMultiPartitionedTableITCase to get rid of managed table
    
    This closes #505
---
 .../CompositePkAndMultiPartitionedTableITCase.java | 1339 ++++++--------------
 ...AndMultiPartitionedTableWIthKafkaLogITCase.java | 1134 +++++++++++++++++
 .../store/connector/ReadWriteTableTestBase.java    |  203 ---
 .../table/store/connector/ShowCreateUtil.java      |   40 -
 .../StreamingReadWriteTableWithKafkaLogITCase.java |  252 ++--
 .../connector/action/CompactActionITCase.java      |   12 +-
 .../connector/util/ReadWriteTableTestUtil.java     |   41 +
 .../table/store/kafka/KafkaTableTestBase.java      |    6 +-
 8 files changed, 1654 insertions(+), 1373 deletions(-)

diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
index a0a6b3f6..693adce9 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
@@ -18,48 +18,46 @@
 
 package org.apache.flink.table.store.connector;
 
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.table.api.EnvironmentSettings;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.table.store.file.utils.BlockingIterator;
+import org.apache.flink.table.store.connector.util.AbstractTestBase;
 import org.apache.flink.types.Row;
 
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
 
 import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyExchangeRates;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyExchangeRatesChangelogWithoutUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithoutUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.exchangeRatesChangelogWithoutUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyExchangeRates;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyExchangeRatesChangelogWithoutUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyRates;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyRatesChangelogWithoutUB;
-import static org.assertj.core.api.Assertions.assertThat;
-
-/**
- * Table store IT case when the managed table has composite primary keys and multiple partition
- * fields.
- */
-public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTestBase {
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.bEnv;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQuery;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildSimpleQuery;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.checkFileStorePath;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTemporaryTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.init;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertIntoFromTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertOverwrite;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertOverwritePartition;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testBatchRead;
+
+/** Table store IT case when the table has composite primary keys and multiple partition fields. */
+public class CompositePkAndMultiPartitionedTableITCase extends AbstractTestBase {
+
+    @BeforeEach
+    public void setUp() {
+        init(getTempDirPath());
+    }
 
     @Test
     public void testBatchWriteWithMultiPartitionedRecordsWithMultiPk() throws Exception {
-        // input is hourlyExchangeRates()
-        List<Row> expectedRecords =
+        List<Row> initialRecords =
                 Arrays.asList(
                         // to_currency is USD, dt = 2022-01-01, hh = 11
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
                         changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
                         changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01", "11"),
                         changelogRow(
                                 "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01", "11"),
                         changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "11"),
@@ -68,69 +66,83 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
                         changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
                         changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "12"),
                         changelogRow(
                                 "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
                         changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12"),
                         // to_currency is Euro, dt = 2022-01-02, hh = 23
                         changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
                         changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23"));
-        // test batch read
-        String managedTable =
-                collectAndCheckBatchReadWrite(
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
                         Arrays.asList("dt", "hh"),
+                        initialRecords,
+                        "dt:2022-01-01,hh:11;dt:2022-01-01,hh:12;dt:2022-01-02,hh:23",
+                        true,
+                        "I");
+
+        String table =
+                createTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
                         Arrays.asList("from_currency", "to_currency", "dt", "hh"),
-                        null,
-                        Collections.emptyList(),
-                        expectedRecords);
-
-        checkFileStorePath(tEnv, managedTable, hourlyExchangeRates().f1);
-
-        // test streaming read
-        final StreamTableEnvironment streamTableEnv =
-                StreamTableEnvironment.create(buildStreamEnv());
-        registerTable(streamTableEnv, managedTable);
-        BlockingIterator<Row, Row> streamIter =
-                collectAndCheck(
-                        streamTableEnv,
-                        managedTable,
-                        Collections.emptyMap(),
-                        null,
-                        expectedRecords);
+                        Arrays.asList("dt", "hh"));
 
-        // overwrite static partition dt = 2022-01-02 and hh = 23
-        Map<String, String> overwritePartition = new LinkedHashMap<>();
-        overwritePartition.put("dt", "'2022-01-02'");
-        overwritePartition.put("hh", "'23'");
-        prepareEnvAndOverwrite(
-                managedTable,
-                overwritePartition,
-                Collections.singletonList(new String[] {"'US Dollar'", "'Thai Baht'", "33.51"}));
+        insertIntoFromTable(temporaryTable, table);
 
-        // streaming iter will not receive any changelog
-        assertNoMoreRecords(streamIter);
+        checkFileStorePath(
+                table,
+                Arrays.asList("dt=2022-01-01,hh=11", "dt=2022-01-01,hh=12", "dt=2022-01-02,hh=23"));
+
+        // test batch read
+        testBatchRead(
+                buildSimpleQuery(table),
+                Arrays.asList(
+                        // to_currency is USD, dt = 2022-01-01, hh = 11
+                        changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
+                        changelogRow(
+                                "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01", "11"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "11"),
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
+                        // to_currency is USD, dt = 2022-01-01, hh = 12
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
+                        changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12"),
+                        changelogRow(
+                                "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12"),
+                        // to_currency is Euro, dt = 2022-01-02, hh = 23
+                        changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
+                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")));
+
+        // overwrite static partition dt = 2022-01-02 and hh = 23
+        insertOverwritePartition(
+                table,
+                "PARTITION (dt = '2022-01-02', hh = '23')",
+                "('US Dollar', 'Thai Baht', 33.51)");
 
         // batch read to check partition refresh
-        collectAndCheck(
-                        tEnv,
-                        managedTable,
-                        Collections.emptyMap(),
-                        "dt = '2022-01-02' AND hh = '23'",
-                        Collections.singletonList(
-                                changelogRow(
-                                        "+I",
-                                        "US Dollar",
-                                        "Thai Baht",
-                                        33.51d,
-                                        "2022-01-02",
-                                        "23")))
-                .close();
+        testBatchRead(
+                buildQuery(table, "*", "WHERE dt = '2022-01-02' AND hh = '23'"),
+                Collections.singletonList(
+                        changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-02", "23")));
 
         // test partition filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "dt = '2022-01-01'",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE dt = '2022-01-01'"),
                 Arrays.asList(
                         // to_currency is USD, dt = 2022-01-01, hh = 11
                         changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
@@ -147,11 +159,8 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                                 "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
                         changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12")));
 
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "dt = '2022-01-01' AND hh = '12'",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE dt = '2022-01-01' AND hh = '12'"),
                 Arrays.asList(
                         changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
                         changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
@@ -161,29 +170,17 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12")));
 
         // test field filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "to_currency = 'Euro'",
-                Collections.emptyList(),
-                Arrays.asList(
-                        changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
-                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")));
+        testBatchRead(
+                buildQuery(table, "*", "WHERE to_currency = 'Euro'"), Collections.emptyList());
 
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "from_currency = 'HK Dollar'",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE from_currency = 'HK Dollar'"),
                 Arrays.asList(
                         changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
                         changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12")));
 
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "rate_by_to_currency > 0.5",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE rate_by_to_currency > 0.5"),
                 Arrays.asList(
                         changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
                         changelogRow(
@@ -193,142 +190,158 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
                         changelogRow(
                                 "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
-                        changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
-                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")));
+                        changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-02", "23")));
 
         // test partition and field filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "rate_by_to_currency > 0.9 AND hh = '23'",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE rate_by_to_currency > 1 AND hh = '12'"),
                 Collections.singletonList(
-                        changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23")));
+                        changelogRow("+I", "Euro", "US Dollar", 1.12, "2022-01-01", "12")));
 
         // test projection
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                null,
-                Arrays.asList("from_currency", "dt", "hh"),
+        testBatchRead(
+                buildQuery(table, "from_currency, dt, hh", ""),
                 Arrays.asList(
-                        // to_currency is USD, dt = 2022-01-01, hh = 11
+                        // dt = 2022-01-01, hh = 11
                         changelogRow("+I", "Euro", "2022-01-01", "11"),
                         changelogRow("+I", "HK Dollar", "2022-01-01", "11"),
                         changelogRow("+I", "Singapore Dollar", "2022-01-01", "11"),
                         changelogRow("+I", "Yen", "2022-01-01", "11"),
                         changelogRow("+I", "US Dollar", "2022-01-01", "11"),
-                        // to_currency is USD, dt = 2022-01-01, hh = 12
+                        // dt = 2022-01-01, hh = 12
                         changelogRow("+I", "US Dollar", "2022-01-01", "12"),
                         changelogRow("+I", "Euro", "2022-01-01", "12"),
                         changelogRow("+I", "HK Dollar", "2022-01-01", "12"),
                         changelogRow("+I", "Singapore Dollar", "2022-01-01", "12"),
                         changelogRow("+I", "Yen", "2022-01-01", "12"),
-                        // to_currency is Euro, dt = 2022-01-02, hh = 23
-                        changelogRow("+I", "US Dollar", "2022-01-02", "23"),
-                        changelogRow("+I", "Singapore Dollar", "2022-01-02", "23")));
+                        // dt = 2022-01-02, hh = 23
+                        changelogRow("+I", "US Dollar", "2022-01-02", "23")));
 
         // test projection and filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "dt = '2022-01-01' AND hh >= '12' OR rate_by_to_currency > 2",
-                Arrays.asList("from_currency", "to_currency"),
+        testBatchRead(
+                buildQuery(
+                        table,
+                        "from_currency, to_currency",
+                        "WHERE dt = '2022-01-01' AND hh >= '12' OR rate_by_to_currency > 2"),
                 Arrays.asList(
-                        // to_currency is USD, dt = 2022-01-01, hh = 12
+                        // dt = 2022-01-01, hh = 12
                         changelogRow("+I", "US Dollar", "US Dollar"),
                         changelogRow("+I", "Euro", "US Dollar"),
                         changelogRow("+I", "HK Dollar", "US Dollar"),
                         changelogRow("+I", "Singapore Dollar", "US Dollar"),
-                        changelogRow("+I", "Yen", "US Dollar")));
+                        changelogRow("+I", "Yen", "US Dollar"),
+                        // dt = 2022-01-02, hh = 23
+                        changelogRow("+I", "US Dollar", "Thai Baht")));
     }
 
     @Test
     public void testBatchWriteWithSinglePartitionedRecordsWithMultiPk() throws Exception {
-        // input is dailyExchangeRates()
-        List<Row> expectedRecords =
+        List<Row> initialRecords =
                 Arrays.asList(
+                        // to_currency is USD
                         changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
                         changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
                         changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
                         changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
                         changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02"),
+                        // to_currency is Euro
                         changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-01"),
                         changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-01"),
+                        // to_currency is Yen
                         changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-01"),
                         changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-01"),
-                        changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01"),
-                        changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
-                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02"));
-        // test batch read
-        String managedTable =
-                collectAndCheckBatchReadWrite(
+                        changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-01"),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01"));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
                         Collections.singletonList("dt"),
+                        initialRecords,
+                        "dt:2022-01-01;dt:2022-01-02",
+                        true,
+                        "I");
+
+        String table =
+                createTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
                         Arrays.asList("from_currency", "to_currency", "dt"),
-                        null,
-                        Collections.emptyList(),
-                        expectedRecords);
-
-        checkFileStorePath(tEnv, managedTable, dailyExchangeRates().f1);
-
-        // test streaming read
-        final StreamTableEnvironment streamTableEnv =
-                StreamTableEnvironment.create(buildStreamEnv());
-        registerTable(streamTableEnv, managedTable);
-        BlockingIterator<Row, Row> streamIter =
-                collectAndCheck(
-                        streamTableEnv,
-                        managedTable,
-                        Collections.emptyMap(),
-                        null,
-                        expectedRecords);
+                        Collections.singletonList("dt"));
+
+        insertIntoFromTable(temporaryTable, table);
+
+        checkFileStorePath(table, Arrays.asList("dt=2022-01-01", "dt=2022-01-02"));
+
+        // test batch read
+        testBatchRead(
+                buildSimpleQuery(table),
+                Arrays.asList(
+                        // to_currency is USD
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+                        changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+                        changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02"),
+                        // to_currency is Euro
+                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-01"),
+                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-01"),
+                        // to_currency is Yen
+                        changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-01"),
+                        changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-01"),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01")));
 
         // overwrite dynamic partition
-        prepareEnvAndOverwrite(
-                managedTable,
-                Collections.emptyMap(),
+        insertOverwrite(table, "('US Dollar', 'Thai Baht', 33.51, '2022-01-01')");
+
+        // batch read to check partition refresh
+        testBatchRead(
+                buildSimpleQuery(table),
                 Collections.singletonList(
-                        new String[] {"'US Dollar'", "'Thai Baht'", "33.51", "'2022-01-01'"}));
+                        changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-01")));
 
-        // streaming iter will not receive any changelog
-        assertNoMoreRecords(streamIter);
+        // reset table
+        table =
+                createTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"));
 
-        // batch read to check partition refresh
-        collectAndCheck(
-                        tEnv,
-                        managedTable,
-                        Collections.emptyMap(),
-                        "dt = '2022-01-01'",
-                        Collections.singletonList(
-                                changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-01")))
-                .close();
+        insertIntoFromTable(temporaryTable, table);
 
         // test partition filter
-        collectAndCheckBatchReadWrite(
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                "dt = '2022-01-02'",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE dt = '2022-01-02'"),
                 Arrays.asList(
                         changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
                         changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02")));
 
         // test field filter
-        collectAndCheckBatchReadWrite(
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                "rate_by_to_currency < 0.1",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE rate_by_to_currency < 0.1"),
                 Arrays.asList(
                         changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
                         changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02")));
 
         // test partition and field filter
-        collectAndCheckBatchReadWrite(
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                "rate_by_to_currency > 0.9 OR dt = '2022-01-02'",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE rate_by_to_currency > 0.9 OR dt = '2022-01-02'"),
                 Arrays.asList(
                         changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
                         changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
@@ -339,17 +352,14 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02")));
 
         // test projection
-        collectAndCheckBatchReadWrite(
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                null,
-                Arrays.asList("rate_by_to_currency", "dt"),
+        testBatchRead(
+                buildQuery(table, "rate_by_to_currency, dt", ""),
                 Arrays.asList(
                         changelogRow("+I", 1.0d, "2022-01-01"), // US Dollar,US Dollar
                         changelogRow("+I", 1.11d, "2022-01-01"), // Euro,US Dollar
                         changelogRow("+I", 0.13d, "2022-01-01"), // HK Dollar,US Dollar
                         changelogRow("+I", 0.0082d, "2022-01-01"), // Yen,US Dollar
-                        changelogRow("+I", 0.74d, "2022-01-01"), // Singapore Dollar,US Dollar
+                        changelogRow("+I", 0.74d, "2022-01-01"), // Singapore Dollar,USDollar
                         changelogRow("+I", 0.9d, "2022-01-01"), // US Dollar,Euro
                         changelogRow("+I", 0.67d, "2022-01-01"), // Singapore Dollar,Euro
                         changelogRow("+I", 1.0d, "2022-01-01"), // Yen,Yen
@@ -360,11 +370,11 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         ));
 
         // test projection and filter
-        collectAndCheckBatchReadWrite(
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                "dt = '2022-01-02' OR rate_by_to_currency > 100",
-                Arrays.asList("from_currency", "to_currency"),
+        testBatchRead(
+                buildQuery(
+                        table,
+                        "from_currency, to_currency",
+                        "WHERE dt = '2022-01-02' OR rate_by_to_currency > 100"),
                 Arrays.asList(
                         changelogRow("+I", "Yen", "US Dollar"),
                         changelogRow("+I", "US Dollar", "US Dollar"),
@@ -373,12 +383,13 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
 
     @Test
     public void testBatchWriteWithNonPartitionedRecordsWithMultiPk() throws Exception {
-        // input is exchangeRates()
-        List<Row> expectedRecords =
+        List<Row> initialRecords =
                 Arrays.asList(
                         // to_currency is USD
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
                         changelogRow("+I", "Euro", "US Dollar", 1.11d),
                         changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0082d),
                         changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
                         changelogRow("+I", "Yen", "US Dollar", 0.0081d),
                         changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
@@ -388,63 +399,61 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         // to_currency is Yen
                         changelogRow("+I", "Yen", "Yen", 1.0d),
                         changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 90.32d),
                         changelogRow("+I", "Singapore Dollar", "Yen", 122.46d));
-        // test batch read
-        String managedTable =
-                collectAndCheckBatchReadWrite(
-                        Collections.emptyList(), // partition
-                        Arrays.asList("from_currency", "to_currency"), // pk
-                        null,
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE"),
+                        Arrays.asList("from_currency", "to_currency"),
                         Collections.emptyList(),
-                        expectedRecords);
-
-        checkFileStorePath(tEnv, managedTable, null);
-
-        // test streaming read
-        final StreamTableEnvironment streamTableEnv =
-                StreamTableEnvironment.create(buildStreamEnv());
-        registerTable(streamTableEnv, managedTable);
-        BlockingIterator<Row, Row> streamIter =
-                collectAndCheck(
-                        streamTableEnv,
-                        managedTable,
-                        Collections.emptyMap(),
+                        initialRecords,
                         null,
-                        expectedRecords);
+                        true,
+                        "I");
 
-        // overwrite the whole table
-        prepareEnvAndOverwrite(
-                managedTable,
-                Collections.emptyMap(),
-                Collections.singletonList(new String[] {"'US Dollar'", "'Thai Baht'", "33.51"}));
+        String table =
+                createTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE"),
+                        Arrays.asList("from_currency", "to_currency"),
+                        Collections.emptyList());
 
-        // streaming iter will not receive any changelog
-        assertNoMoreRecords(streamIter);
+        insertIntoFromTable(temporaryTable, table);
 
-        // batch read to check data refresh
-        collectAndCheck(
-                        tEnv,
-                        managedTable,
-                        Collections.emptyMap(),
-                        null,
-                        Collections.singletonList(
-                                changelogRow("+I", "US Dollar", "Thai Baht", 33.51d)))
-                .close();
+        checkFileStorePath(table, Collections.emptyList());
+
+        // test batch read
+        testBatchRead(
+                buildSimpleQuery(table),
+                Arrays.asList(
+                        // to_currency is USD
+                        changelogRow("+I", "Euro", "US Dollar", 1.11d),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+                        changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0081d),
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
+                        // to_currency is Euro
+                        changelogRow("+I", "US Dollar", "Euro", 0.9d),
+                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+                        // to_currency is Yen
+                        changelogRow("+I", "Yen", "Yen", 1.0d),
+                        changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 122.46d)));
 
         // test field filter
-        collectAndCheckBatchReadWrite(
-                Collections.emptyList(), // partition
-                Arrays.asList("from_currency", "to_currency"), // pk
-                "rate_by_to_currency < 0.1",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE rate_by_to_currency < 0.1"),
                 Collections.singletonList(changelogRow("+I", "Yen", "US Dollar", 0.0081d)));
 
         // test projection
-        collectAndCheckBatchReadWrite(
-                Collections.emptyList(), // partition
-                Arrays.asList("from_currency", "to_currency"), // pk
-                null,
-                Collections.singletonList("rate_by_to_currency"),
+        testBatchRead(
+                buildQuery(table, "rate_by_to_currency", ""),
                 Arrays.asList(
                         changelogRow("+I", 1.11d), // Euro, US Dollar
                         changelogRow("+I", 0.13d), // HK Dollar, US Dollar
@@ -459,86 +468,60 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         ));
 
         // test projection and filter
-        collectAndCheckBatchReadWrite(
-                Collections.emptyList(), // partition
-                Arrays.asList("from_currency", "to_currency"), // pk
-                "rate_by_to_currency > 100",
-                Arrays.asList("from_currency", "to_currency"),
+        testBatchRead(
+                buildQuery(table, "from_currency, to_currency", "WHERE rate_by_to_currency > 100"),
                 Collections.singletonList(changelogRow("+I", "Singapore Dollar", "Yen")));
     }
 
     @Test
     public void testBatchWriteMultiPartitionedRecordsWithOnePk() throws Exception {
-        // input is hourlyRates()
-        List<Row> expectedRecords =
+        List<Row> initialRecords =
                 Arrays.asList(
                         // dt = 2022-01-01, hh = 00
+                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "00"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
                         changelogRow("+I", "Yen", 1L, "2022-01-01", "00"),
                         changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
                         changelogRow("+I", "US Dollar", 114L, "2022-01-01", "00"),
                         // dt = 2022-01-01, hh = 20
+                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "20"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
                         changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
                         changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
                         changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
                         // dt = 2022-01-02, hh = 12
                         changelogRow("+I", "Euro", 119L, "2022-01-02", "12"));
-        // test batch read
-        String managedTable =
-                collectAndCheckBatchReadWrite(
-                        Arrays.asList("dt", "hh"), // partition
-                        Arrays.asList("currency", "dt", "hh"), // pk
-                        null,
-                        Collections.emptyList(),
-                        expectedRecords);
-
-        checkFileStorePath(tEnv, managedTable, hourlyRates().f1);
-
-        // test streaming read
-        final StreamTableEnvironment streamTableEnv =
-                StreamTableEnvironment.create(buildStreamEnv());
-        registerTable(streamTableEnv, managedTable);
-        BlockingIterator<Row, Row> streamIter =
-                collectAndCheck(
-                        streamTableEnv,
-                        managedTable,
-                        Collections.emptyMap(),
-                        null,
-                        expectedRecords);
 
-        // dynamic overwrite
-        // INSERT OVERWRITE `manged_table-${uuid}` VALUES(...)
-        prepareEnvAndOverwrite(
-                managedTable,
-                Collections.emptyMap(),
-                Collections.singletonList(
-                        new String[] {"'HK Dollar'", "80", "'2022-01-01'", "'00'"}));
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Arrays.asList("currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        initialRecords,
+                        "dt:2022-01-01,hh:00;dt:2022-01-01,hh:20;dt:2022-01-02,hh:12",
+                        true,
+                        "I");
 
-        // INSERT OVERWRITE `manged_table-${uuid}` PARTITION (dt = '2022-01-02') VALUES(...)
-        prepareEnvAndOverwrite(
-                managedTable,
-                Collections.singletonMap("dt", "'2022-01-02'"),
-                Collections.singletonList(new String[] {"'Euro'", "120", "'12'"}));
+        String table =
+                createTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Arrays.asList("currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"));
 
-        // batch read to check data refresh
-        collectAndCheck(
-                tEnv,
-                managedTable,
-                Collections.emptyMap(),
-                "hh = '00' OR hh = '12'",
-                Arrays.asList(
-                        changelogRow("+I", "HK Dollar", 80L, "2022-01-01", "00"),
-                        changelogRow("+I", "Euro", 120L, "2022-01-02", "12")));
+        insertIntoFromTable(temporaryTable, table);
 
-        // streaming iter will not receive any changelog
-        assertNoMoreRecords(streamIter);
+        checkFileStorePath(
+                table,
+                Arrays.asList("dt=2022-01-01,hh=00", "dt=2022-01-01,hh=20", "dt=2022-01-02,hh=12"));
 
-        // test partition filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("currency", "dt", "hh"), // pk
-                "hh >= '10'",
-                Collections.emptyList(),
+        // test batch read
+        testBatchRead(
+                buildSimpleQuery(table),
                 Arrays.asList(
+                        // dt = 2022-01-01, hh = 00
+                        changelogRow("+I", "Yen", 1L, "2022-01-01", "00"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+                        changelogRow("+I", "US Dollar", 114L, "2022-01-01", "00"),
                         // dt = 2022-01-01, hh = 20
                         changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
                         changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
@@ -546,22 +529,23 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         // dt = 2022-01-02, hh = 12
                         changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
 
-        // test field filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("currency", "dt", "hh"), // pk
-                "rate >= 119",
-                Collections.emptyList(),
-                Collections.singletonList(
-                        // dt = 2022-01-02, hh = 12
+        // test partition filter
+        testBatchRead(
+                buildQuery(table, "*", "WHERE hh >= '10'"),
+                Arrays.asList(
+                        changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+                        changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
                         changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
 
+        // test field filter
+        testBatchRead(
+                buildQuery(table, "*", "WHERE rate >= 119"),
+                Collections.singletonList(changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+
         // test projection
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("currency", "dt", "hh"), // pk
-                null,
-                Collections.singletonList("hh"),
+        testBatchRead(
+                buildQuery(table, "hh", ""),
                 Arrays.asList(
                         changelogRow("+I", "00"), // Yen, 1L, 2022-01-01
                         changelogRow("+I", "00"), // Euro, 114L, 2022-01-01
@@ -573,93 +557,98 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         ));
 
         // test projection and filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("currency", "dt", "hh"), // pk
-                "rate > 100 AND hh >= '20'",
-                Collections.singletonList("rate"),
-                Collections.singletonList(changelogRow("+I", 114L)));
+        testBatchRead(
+                buildQuery(table, "rate", "WHERE rate > 100 AND hh >= '20'"),
+                Collections.nCopies(2, changelogRow("+I", 114L)));
     }
 
     @Test
     public void testBatchWriteMultiPartitionedRecordsWithoutPk() throws Exception {
-        // input is hourlyRates()
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // dt = 2022-01-01, hh = 00
+                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "00"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+                        changelogRow("+I", "Yen", 1L, "2022-01-01", "00"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+                        changelogRow("+I", "US Dollar", 114L, "2022-01-01", "00"),
+                        // dt = 2022-01-01, hh = 20
+                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "20"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+                        changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+                        changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
+                        // dt = 2022-01-02, hh = 12
+                        changelogRow("+I", "Euro", 119L, "2022-01-02", "12"));
 
-        // test batch read
-        String managedTable =
-                collectAndCheckBatchReadWrite(
-                        Arrays.asList("dt", "hh"), // partition
-                        Collections.emptyList(), // pk
-                        null,
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
                         Collections.emptyList(),
-                        hourlyRates().f0);
-
-        checkFileStorePath(tEnv, managedTable, hourlyRates().f1);
-
-        // test streaming read
-        final StreamTableEnvironment streamTableEnv =
-                StreamTableEnvironment.create(buildStreamEnv());
-        registerTable(streamTableEnv, managedTable);
-        BlockingIterator<Row, Row> streamIter =
-                collectAndCheck(
-                        streamTableEnv,
-                        managedTable,
-                        Collections.emptyMap(),
-                        null,
-                        hourlyRates().f0);
+                        Arrays.asList("dt", "hh"),
+                        initialRecords,
+                        "dt:2022-01-01,hh:00;dt:2022-01-01,hh:20;dt:2022-01-02,hh:12",
+                        true,
+                        "I");
+
+        String table =
+                createTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"));
+
+        insertIntoFromTable(temporaryTable, table);
+
+        checkFileStorePath(
+                table,
+                Arrays.asList("dt=2022-01-01,hh=00", "dt=2022-01-01,hh=20", "dt=2022-01-02,hh=12"));
+
+        // test batch read
+        testBatchRead(buildSimpleQuery(table), initialRecords);
 
         // dynamic overwrite the whole table with null partitions
-        String query =
-                String.format(
-                        "INSERT OVERWRITE `%s` SELECT 'Yen', 1, CAST(null AS STRING), CAST(null AS STRING) FROM `%s`",
-                        managedTable, managedTable);
-        prepareEnvAndOverwrite(managedTable, query);
+        bEnv.executeSql(
+                        String.format(
+                                "INSERT OVERWRITE `%s` SELECT 'Yen', 1, CAST(NULL AS STRING), CAST(NULL AS STRING) FROM `%s`",
+                                table, table))
+                .await();
+
         // check new partition path
-        checkFileStorePath(tEnv, managedTable, "dt:null,hh:null");
+        checkFileStorePath(table, Collections.singletonList("dt=null,hh=null"));
 
         // batch read to check data refresh
-        collectAndCheck(
-                tEnv,
-                managedTable,
-                Collections.emptyMap(),
-                null,
-                Collections.singletonList(changelogRow("+I", "Yen", 1L, null, null)));
+        testBatchRead(
+                buildSimpleQuery(table),
+                Collections.nCopies(11, changelogRow("+I", "Yen", 1L, null, null)));
+
+        // reset table
+        table =
+                createTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"));
 
-        // streaming iter will not receive any changelog
-        assertNoMoreRecords(streamIter);
+        insertIntoFromTable(temporaryTable, table);
 
         // test partition filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Collections.emptyList(), // pk
-                "hh >= '10'",
-                Collections.emptyList(),
+        testBatchRead(
+                buildQuery(table, "*", "WHERE hh >= '10'"),
                 Arrays.asList(
-                        // dt = 2022-01-01, hh = 20
                         changelogRow("+I", "US Dollar", 102L, "2022-01-01", "20"),
                         changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
                         changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
                         changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
                         changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
-                        // dt = 2022-01-02, hh = 12
                         changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
 
         // test field filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Collections.emptyList(), // pk
-                "rate >= 119",
-                Collections.emptyList(),
-                Collections.singletonList(
-                        // dt = 2022-01-02, hh = 12
-                        changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+        testBatchRead(
+                buildQuery(table, "*", "WHERE rate >= 119"),
+                Collections.singletonList(changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
 
         // test projection
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Collections.emptyList(), // pk
-                null,
-                Collections.singletonList("currency"),
+        testBatchRead(
+                buildQuery(table, "currency", ""),
                 Arrays.asList(
                         // dt = 2022-01-01, hh = 00
                         changelogRow("+I", "US Dollar"),
@@ -677,610 +666,8 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
                         changelogRow("+I", "Euro")));
 
         // test projection and filter
-        collectAndCheckBatchReadWrite(
-                Arrays.asList("dt", "hh"), // partition
-                Collections.emptyList(), // pk
-                "rate > 100 AND hh >= '20'",
-                Collections.singletonList("rate"),
-                Collections.singletonList(changelogRow("+I", 114L)));
-    }
-
-    @Test
-    public void testEnableLogAndStreamingReadWriteSinglePartitionedRecordsWithMultiPk()
-            throws Exception {
-        // input is dailyExchangeRatesChangelogWithoutUB()
-        // test hybrid read
-        Tuple2<String, BlockingIterator<Row, Row>> tuple =
-                collectAndCheckStreamingReadWriteWithoutClose(
-                        Collections.singletonList("dt"),
-                        Arrays.asList("from_currency", "to_currency", "dt"),
-                        Collections.emptyMap(),
-                        null,
-                        Collections.emptyList(),
-                        Arrays.asList(
-                                changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
-                                changelogRow(
-                                        "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
-                                changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
-                                changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
-                                changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
-                                changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
-                                changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
-                                changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
-                                changelogRow(
-                                        "+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-02")));
-        String managedTable = tuple.f0;
-        checkFileStorePath(tEnv, managedTable, dailyRatesChangelogWithoutUB().f1);
-        BlockingIterator<Row, Row> streamIter = tuple.f1;
-
-        // test streaming consume changelog
-        tEnv.executeSql(
-                        String.format(
-                                "INSERT INTO `%s` PARTITION (dt = '2022-01-03')\n"
-                                        + "VALUES('Chinese Yuan', 'HK Dollar', 1.231)\n",
-                                managedTable))
-                .await();
-
-        assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
-                .containsExactlyInAnyOrderElementsOf(
-                        Collections.singletonList(
-                                changelogRow(
-                                        "+I", "Chinese Yuan", "HK Dollar", 1.231d, "2022-01-03")));
-
-        // dynamic overwrite the whole table
-        String query =
-                String.format(
-                        "INSERT OVERWRITE `%s` SELECT 'US Dollar', 'US Dollar', 1, '2022-04-02' FROM `%s`",
-                        managedTable, managedTable);
-        prepareEnvAndOverwrite(managedTable, query);
-        checkFileStorePath(tEnv, managedTable, "dt:2022-04-02");
-
-        // batch read to check data refresh
-        final StreamTableEnvironment batchTableEnv =
-                StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
-        registerTable(batchTableEnv, managedTable);
-        collectAndCheck(
-                batchTableEnv,
-                managedTable,
-                Collections.emptyMap(),
-                null,
-                Collections.singletonList(
-                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-04-02")));
-
-        // check no changelog generated for streaming read
-        assertNoMoreRecords(streamIter);
-
-        // filter on partition and field filter
-        collectAndCheckStreamingReadWriteWithClose(
-                true,
-                Collections.singletonList("dt"),
-                Arrays.asList("from_currency", "to_currency", "dt"),
-                Collections.emptyMap(),
-                "dt = '2022-01-02' AND from_currency = 'US Dollar'",
-                Collections.emptyList(),
-                Collections.singletonList(
-                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02")));
-
-        // test projection and filter
-        collectAndCheckStreamingReadWriteWithClose(
-                true,
-                Collections.singletonList("dt"),
-                Arrays.asList("from_currency", "to_currency", "dt"),
-                Collections.emptyMap(),
-                "dt = '2022-01-01' AND rate_by_to_currency IS NULL",
-                Arrays.asList("from_currency", "to_currency"),
-                Collections.emptyList());
-    }
-
-    @Test
-    public void testEnableLogAndStreamingReadWriteMultiPartitionedRecordsWithMultiPk()
-            throws Exception {
-        // input is hourlyExchangeRatesChangelogWithoutUB()
-        // test hybrid read
-        Tuple2<String, BlockingIterator<Row, Row>> tuple =
-                collectAndCheckStreamingReadWriteWithoutClose(
-                        Arrays.asList("dt", "hh"),
-                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
-                        Collections.emptyMap(),
-                        null,
-                        Collections.emptyList(),
-                        Arrays.asList(
-                                changelogRow(
-                                        "+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
-                                changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
-                                changelogRow(
-                                        "+I",
-                                        "Singapore Dollar",
-                                        "US Dollar",
-                                        0.76d,
-                                        "2022-01-02",
-                                        "20"),
-                                changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
-                                changelogRow(
-                                        "+I", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
-                                changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
-                                changelogRow(
-                                        "+I", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
-                                changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
-                                changelogRow(
-                                        "+I",
-                                        "Singapore Dollar",
-                                        "Yen",
-                                        90.1d,
-                                        "2022-01-02",
-                                        "20")));
-        String managedTable = tuple.f0;
-        checkFileStorePath(tEnv, managedTable, hourlyExchangeRatesChangelogWithoutUB().f1);
-        BlockingIterator<Row, Row> streamIter = tuple.f1;
-
-        // test streaming consume changelog
-        tEnv.executeSql(
-                        String.format(
-                                "INSERT INTO `%s` \n"
-                                        + "VALUES('Chinese Yuan', 'HK Dollar', 1.231, '2022-01-03', '15')\n",
-                                managedTable))
-                .await();
-
-        assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
-                .containsExactlyInAnyOrderElementsOf(
-                        Collections.singletonList(
-                                changelogRow(
-                                        "+I",
-                                        "Chinese Yuan",
-                                        "HK Dollar",
-                                        1.231d,
-                                        "2022-01-03",
-                                        "15")));
-
-        // dynamic overwrite the whole table
-        String query =
-                String.format(
-                        "INSERT OVERWRITE `%s` SELECT 'US Dollar', 'US Dollar', 1, '2022-04-02', '10' FROM `%s`",
-                        managedTable, managedTable);
-        prepareEnvAndOverwrite(managedTable, query);
-        checkFileStorePath(tEnv, managedTable, "dt:2022-04-02,hh:10");
-
-        // batch read to check data refresh
-        final StreamTableEnvironment batchTableEnv =
-                StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
-        registerTable(batchTableEnv, managedTable);
-        collectAndCheck(
-                batchTableEnv,
-                managedTable,
-                Collections.emptyMap(),
-                null,
-                Collections.singletonList(
-                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-04-02", "10")));
-
-        // check no changelog generated for streaming read
-        assertNoMoreRecords(streamIter);
-
-        // filter on partition and field filter
-        collectAndCheckStreamingReadWriteWithClose(
-                true,
-                Arrays.asList("dt", "hh"),
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"),
-                Collections.emptyMap(),
-                "dt = '2022-01-02' AND from_currency = 'US Dollar'",
-                Collections.emptyList(),
-                Arrays.asList(
-                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
-                        changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
-
-        // test projection and filter
-        collectAndCheckStreamingReadWriteWithClose(
-                true,
-                Arrays.asList("dt", "hh"),
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"),
-                Collections.emptyMap(),
-                "dt = '2022-01-02' AND from_currency = 'US Dollar'",
-                Arrays.asList("from_currency", "to_currency"),
-                Arrays.asList(
-                        changelogRow("+I", "US Dollar", "Euro"),
-                        changelogRow("+I", "US Dollar", "Yen")));
-    }
-
-    @Test
-    public void testEnableLogAndStreamingReadWriteMultiPartitionedRecordsWithoutPk()
-            throws Exception {
-        // input is dailyExchangeRatesChangelogWithUB()
-        // test hybrid read
-        Tuple2<String, BlockingIterator<Row, Row>> tuple =
-                collectAndCheckStreamingReadWriteWithoutClose(
-                        Collections.singletonList("dt"),
-                        Collections.emptyList(),
-                        Collections.emptyMap(),
-                        null,
-                        Collections.emptyList(),
-                        Arrays.asList(
-                                // dt = 2022-01-01
-                                changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
-                                // dt = 2022-01-02
-                                changelogRow("+I", "Euro", 115L, "2022-01-02")));
-        String managedTable = tuple.f0;
-        checkFileStorePath(tEnv, managedTable, dailyRatesChangelogWithUB().f1);
-        BlockingIterator<Row, Row> streamIter = tuple.f1;
-
-        // test streaming consume changelog
-        tEnv.executeSql(
-                        String.format(
-                                "INSERT INTO `%s` PARTITION (dt = '2022-04-02')\n"
-                                        + "VALUES('Euro', 116)\n",
-                                managedTable))
-                .await();
-
-        assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
-                .containsExactlyInAnyOrderElementsOf(
-                        Collections.singletonList(changelogRow("+I", "Euro", 116L, "2022-04-02")));
-
-        // dynamic overwrite the whole table
-        String query =
-                String.format(
-                        "INSERT OVERWRITE `%s` SELECT 'US Dollar', 103, '2022-04-02' FROM `%s`",
-                        managedTable, managedTable);
-        prepareEnvAndOverwrite(managedTable, query);
-        checkFileStorePath(tEnv, managedTable, "dt:2022-04-02");
-
-        // batch read to check data refresh
-        final StreamTableEnvironment batchTableEnv =
-                StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
-        registerTable(batchTableEnv, managedTable);
-        collectAndCheck(
-                batchTableEnv,
-                managedTable,
-                Collections.emptyMap(),
-                null,
-                Collections.singletonList(changelogRow("+I", "US Dollar", 103L, "2022-04-02")));
-
-        // check no changelog generated for streaming read
-        assertNoMoreRecords(streamIter);
-
-        // filter on partition and field filter
-        collectAndCheckStreamingReadWriteWithClose(
-                true,
-                Collections.singletonList("dt"),
-                Collections.emptyList(),
-                Collections.emptyMap(),
-                "dt = '2022-01-01' AND currency = 'Yen'",
-                Collections.emptyList(),
-                Collections.emptyList());
-
-        // test projection and filter
-        collectAndCheckStreamingReadWriteWithClose(
-                true,
-                Collections.singletonList("dt"),
-                Collections.emptyList(),
-                Collections.emptyMap(),
-                "dt = '2022-01-01' AND rate = 103",
-                Collections.singletonList("currency"),
-                Collections.emptyList());
-    }
-
-    @Test
-    public void testReadLatestChangelogOfMultiPartitionedRecordsWithMultiPk() throws Exception {
-        // input is hourlyExchangeRatesChangelogWithoutUB()
-        List<Row> expectedRecords = new ArrayList<>(hourlyExchangeRatesChangelogWithoutUB().f0);
-        expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"));
-        expectedRecords.add(
-                changelogRow("-U", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"));
-        expectedRecords.add(
-                changelogRow("-U", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"));
-        expectedRecords.add(changelogRow("-U", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"));
-        expectedRecords.add(
-                changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"));
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                null,
-                Collections.emptyList(),
-                expectedRecords);
-
-        // test partition filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "dt = '2022-01-02' AND hh = '21'",
-                Collections.emptyList(),
-                Collections.singletonList(
-                        changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
-
-        // test field filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"),
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "rate_by_to_currency IS NOT NULL AND from_currency = 'US Dollar'",
-                Collections.emptyList(),
-                Arrays.asList(
-                        // to_currency is USD
-                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
-                        changelogRow("-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
-                        // to_currency is Euro
-                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
-                        // to_currency is Yen
-                        changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
-
-        // test partition and field filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"),
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "hh = '21' AND from_currency = 'US Dollar'",
-                Collections.emptyList(),
-                Collections.singletonList(
-                        changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
-
-        // test projection
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"),
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                null,
-                Arrays.asList("from_currency", "to_currency"),
-                Arrays.asList(
-                        // to_currency is USD
-                        changelogRow("+I", "US Dollar", "US Dollar"),
-                        changelogRow("+I", "Euro", "US Dollar"),
-                        changelogRow("+I", "HK Dollar", "US Dollar"),
-                        changelogRow("+I", "Yen", "US Dollar"),
-                        changelogRow("+I", "Singapore Dollar", "US Dollar"),
-                        changelogRow("-D", "US Dollar", "US Dollar"),
-                        changelogRow("-D", "Euro", "US Dollar"),
-                        // to_currency is Euro
-                        changelogRow("+I", "US Dollar", "Euro"),
-                        changelogRow("+I", "Singapore Dollar", "Euro"),
-                        // to_currency is Yen
-                        changelogRow("+I", "Yen", "Yen"),
-                        changelogRow("+I", "Chinese Yuan", "Yen"),
-                        changelogRow("+I", "Singapore Dollar", "Yen"),
-                        changelogRow("+I", "US Dollar", "Yen")));
-
-        // test projection and filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"),
-                Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
-                "rate_by_to_currency > 100",
-                Arrays.asList("from_currency", "to_currency"),
-                Collections.singletonList(changelogRow("+I", "US Dollar", "Yen")));
-    }
-
-    @Test
-    public void testReadLatestChangelogOfSinglePartitionedRecordsWithMultiPk() throws Exception {
-        // input is dailyExchangeRatesChangelogWithoutUB()
-        List<Row> expectedRecords = new ArrayList<>(dailyExchangeRatesChangelogWithoutUB().f0);
-        expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"));
-        expectedRecords.add(changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01"));
-        expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"));
-
-        collectLatestLogAndCheck(
-                false,
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                null,
-                Collections.emptyList(),
-                expectedRecords);
-
-        // test partition filter
-        collectLatestLogAndCheck(
-                false,
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                "dt = '2022-01-02'",
-                Collections.emptyList(),
-                Arrays.asList(
-                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
-                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
-                        changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
-                        changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
-                        changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
-                        changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
-                        changelogRow("+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"),
-                        changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02")));
-
-        // test field filter
-        collectLatestLogAndCheck(
-                false,
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                "rate_by_to_currency IS NULL",
-                Collections.emptyList(),
-                Arrays.asList(
-                        changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
-                        changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
-                        changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"),
-                        changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01")));
-
-        // test partition and field filter
-        collectLatestLogAndCheck(
-                false,
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                "dt = '2022-01-02' AND from_currency = 'Yen'",
-                Collections.emptyList(),
-                Collections.singletonList(changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02")));
-
-        // test projection and filter
-        collectLatestLogAndCheck(
-                false,
-                Collections.singletonList("dt"), // partition
-                Arrays.asList("from_currency", "to_currency", "dt"), // pk
-                "rate_by_to_currency > 100",
-                Arrays.asList("from_currency", "to_currency"),
-                Collections.singletonList(changelogRow("+U", "Singapore Dollar", "Yen")));
-    }
-
-    @Test
-    public void testReadLatestChangelogOfNonPartitionedRecordsWithMultiPk() throws Exception {
-        // input is exchangeRatesChangelogWithoutUB()
-        List<Row> expectedRecords = new ArrayList<>(exchangeRatesChangelogWithoutUB());
-        expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d));
-        expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", 1.11d));
-        expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Euro", 0.67d));
-        expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d));
-        expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 122.46d));
-
-        collectLatestLogAndCheck(
-                false,
-                Collections.emptyList(), // partition
-                Arrays.asList("from_currency", "to_currency"), // pk
-                null,
-                Collections.emptyList(),
-                expectedRecords);
-
-        // test field filter
-        collectLatestLogAndCheck(
-                false,
-                Collections.emptyList(), // partition
-                Arrays.asList("from_currency", "to_currency"), // pk
-                "rate_by_to_currency < 1 OR rate_by_to_currency > 100",
-                Collections.emptyList(),
-                Arrays.asList(
-                        // to_currency is USD
-                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
-                        changelogRow("+I", "Yen", "US Dollar", 0.0082d),
-                        changelogRow("-U", "Yen", "US Dollar", 0.0082d),
-                        changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
-                        changelogRow("+U", "Yen", "US Dollar", 0.0081d),
-                        changelogRow("-D", "Yen", "US Dollar", 0.0081d),
-                        // to_currency is Euro
-                        changelogRow("+I", "US Dollar", "Euro", 0.9d),
-                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
-                        changelogRow("-U", "Singapore Dollar", "Euro", 0.67d),
-                        changelogRow("+U", "Singapore Dollar", "Euro", 0.69d),
-                        // to_currency is Yen
-                        changelogRow("+U", "Singapore Dollar", "Yen", 122.46d),
-                        changelogRow("-U", "Singapore Dollar", "Yen", 122.46d),
-                        changelogRow("+U", "Singapore Dollar", "Yen", 122d)));
-
-        // test projection and filter
-        collectLatestLogAndCheck(
-                false,
-                Collections.emptyList(), // partition
-                Arrays.asList("from_currency", "to_currency"), // pk
-                "rate_by_to_currency < 1 OR rate_by_to_currency > 100",
-                Arrays.asList("from_currency", "to_currency"),
-                Arrays.asList(
-                        // to_currency is USD
-                        changelogRow("+I", "HK Dollar", "US Dollar"),
-                        changelogRow("+I", "Yen", "US Dollar"),
-                        changelogRow("-U", "Yen", "US Dollar"),
-                        changelogRow("+I", "Singapore Dollar", "US Dollar"),
-                        changelogRow("+U", "Yen", "US Dollar"),
-                        changelogRow("-D", "Yen", "US Dollar"),
-                        // to_currency is Euro
-                        changelogRow("+I", "US Dollar", "Euro"),
-                        changelogRow("+I", "Singapore Dollar", "Euro"),
-                        changelogRow("-U", "Singapore Dollar", "Euro"),
-                        changelogRow("+U", "Singapore Dollar", "Euro"),
-                        // to_currency is Yen
-                        changelogRow("+U", "Singapore Dollar", "Yen"),
-                        changelogRow("-U", "Singapore Dollar", "Yen"),
-                        changelogRow("+U", "Singapore Dollar", "Yen")));
-    }
-
-    @Test
-    public void testReadLatestChangelogOfMultiPartitionedRecordsWithOnePk() throws Exception {
-        // input is hourlyRatesChangelogWithoutUB()
-        List<Row> expectedRecords = new ArrayList<>(hourlyRatesChangelogWithoutUB().f0);
-        expectedRecords.remove(changelogRow("+U", "Euro", 119L, "2022-01-02", "23"));
-        expectedRecords.add(changelogRow("-U", "Euro", 114L, "2022-01-01", "15"));
-
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("currency", "dt", "hh"), // pk
-                null,
-                Collections.emptyList(),
-                expectedRecords);
-
-        // test partition filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("currency", "dt", "hh"), // pk
-                "dt >= '2022-01-02'",
-                Collections.emptyList(),
-                Collections.singletonList(changelogRow("+I", "Euro", 119L, "2022-01-02", "23")));
-
-        // test field filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("currency", "dt", "hh"), // pk
-                "rate = 1",
-                Collections.emptyList(),
-                Arrays.asList(
-                        changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
-                        changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
-
-        // test projection and filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Arrays.asList("currency", "dt", "hh"), // pk
-                "rate = 1",
-                Collections.singletonList("currency"),
-                Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")));
-    }
-
-    @Test
-    public void testReadLatestChangelogOfMultiPartitionedRecordsWithoutPk() throws Exception {
-        // input is hourlyRatesChangelogWithUB()
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Collections.emptyList(), // pk
-                null,
-                Collections.emptyList(),
-                Arrays.asList(
-                        // dt = 2022-01-01, hh = 15
-                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
-                        changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
-                        changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
-                        changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
-                        changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
-                        // dt = 2022-01-02, hh = 20
-                        changelogRow("+I", "Euro", 114L, "2022-01-02", "20"),
-                        changelogRow("-D", "Euro", 114L, "2022-01-02", "20"),
-                        changelogRow("+I", "Euro", 119L, "2022-01-02", "20"),
-                        changelogRow("-D", "Euro", 119L, "2022-01-02", "20"),
-                        changelogRow("+I", "Euro", 115L, "2022-01-02", "20")));
-
-        // test partition filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Collections.emptyList(), // pk
-                "hh <> '20'",
-                Collections.emptyList(),
-                Arrays.asList(
-                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
-                        changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
-                        changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
-                        changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
-                        changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
-
-        // test field filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Collections.emptyList(), // pk
-                "rate = 1",
-                Collections.emptyList(),
-                Arrays.asList(
-                        changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
-                        changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
-
-        // test projection and filter
-        collectLatestLogAndCheck(
-                false,
-                Arrays.asList("dt", "hh"), // partition
-                Collections.emptyList(), // pk
-                "rate = 1",
-                Collections.singletonList("currency"),
-                Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")));
+        testBatchRead(
+                buildQuery(table, "rate", "WHERE rate > 110 AND hh >= '20'"),
+                Collections.nCopies(3, changelogRow("+I", 114L)));
     }
 }
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableWIthKafkaLogITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableWIthKafkaLogITCase.java
new file mode 100644
index 00000000..0192e4ec
--- /dev/null
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableWIthKafkaLogITCase.java
@@ -0,0 +1,1134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.connector;
+
+import org.apache.flink.table.store.file.utils.BlockingIterator;
+import org.apache.flink.table.store.kafka.KafkaTableTestBase;
+import org.apache.flink.types.Row;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.SCAN_LATEST;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.assertNoMoreRecords;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.bEnv;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQuery;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQueryWithTableOptions;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildSimpleQuery;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.checkFileStorePath;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTableWithKafkaLog;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTemporaryTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.init;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertInto;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertIntoFromTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertIntoPartition;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testBatchRead;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingRead;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingReadWithReadFirst;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.validateStreamingReadResult;
+
+/**
+ * IT cases of streaming reading and writing tables which have composite primary keys and multiple
+ * partition fields with Kafka log.
+ */
+public class CompositePkAndMultiPartitionedTableWIthKafkaLogITCase extends KafkaTableTestBase {
+
+    @Before
+    public void setUp() throws Exception {
+        init(createAndRegisterTempFile("").toString());
+    }
+
+    // ----------------------------------------------------------------------------------------------------------------
+    // Non latest
+    // ----------------------------------------------------------------------------------------------------------------
+
+    @Test
+    public void testStreamingReadWriteMultiPartitionedRecordsWithMultiPk() throws Exception {
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // to_currency is USD
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+                        changelogRow("+I", "Euro", "US Dollar", null, "2022-01-02", "20"),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"),
+                        changelogRow(
+                                "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"),
+                        changelogRow("+U", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
+                        changelogRow("-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+                        changelogRow("-D", "Euro", "US Dollar", null, "2022-01-02", "20"),
+                        changelogRow(
+                                "+U", "Singapore Dollar", "US Dollar", 0.76d, "2022-01-02", "20"),
+                        // to_currency is Euro
+                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"),
+                        changelogRow("+U", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
+                        // to_currency is Yen
+                        changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
+                        changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"),
+                        changelogRow("+U", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"),
+                        changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
+                        changelogRow("+U", "Singapore Dollar", "Yen", 90.1d, "2022-01-02", "20"));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        initialRecords,
+                        "dt:2022-01-02,hh:20;dt:2022-01-02,hh:21",
+                        false,
+                        "I,UA,D");
+
+        String table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        false);
+
+        insertIntoFromTable(temporaryTable, table);
+
+        checkFileStorePath(table, Arrays.asList("dt=2022-01-02,hh=20", "dt=2022-01-02,hh=21"));
+
+        BlockingIterator<Row, Row> streamingItr =
+                testStreamingRead(
+                        buildSimpleQuery(table),
+                        Arrays.asList(
+                                changelogRow(
+                                        "+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
+                                changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
+                                changelogRow(
+                                        "+I",
+                                        "Singapore Dollar",
+                                        "US Dollar",
+                                        0.76d,
+                                        "2022-01-02",
+                                        "20"),
+                                changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+                                changelogRow(
+                                        "+I", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
+                                changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
+                                changelogRow(
+                                        "+I", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
+                                changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
+                                changelogRow(
+                                        "+I",
+                                        "Singapore Dollar",
+                                        "Yen",
+                                        90.1d,
+                                        "2022-01-02",
+                                        "20")));
+
+        // test streaming consume changelog
+        insertInto(table, "('Chinese Yuan', 'HK Dollar', 1.231, '2022-01-03', '15')");
+
+        validateStreamingReadResult(
+                streamingItr,
+                Collections.singletonList(
+                        changelogRow(
+                                "+I", "Chinese Yuan", "HK Dollar", 1.231d, "2022-01-03", "15")));
+
+        // dynamic overwrite the whole table
+        bEnv.executeSql(
+                        String.format(
+                                "INSERT OVERWRITE `%s` SELECT 'US Dollar', 'US Dollar', 1, '2022-04-02', '10' FROM `%s`",
+                                table, table))
+                .await();
+
+        checkFileStorePath(table, Collections.singletonList("dt=2022-04-02,hh=10"));
+
+        // batch read to check data refresh
+        testBatchRead(
+                buildSimpleQuery(table),
+                Collections.singletonList(
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-04-02", "10")));
+
+        // check no changelog generated for streaming read
+        assertNoMoreRecords(streamingItr);
+        streamingItr.close();
+
+        // reset table
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        false);
+
+        insertIntoFromTable(temporaryTable, table);
+
+        // filter on partition and field filter
+        testStreamingRead(
+                        buildQuery(
+                                table,
+                                "*",
+                                "WHERE dt = '2022-01-02' AND from_currency = 'US Dollar'"),
+                        Arrays.asList(
+                                changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+                                changelogRow(
+                                        "+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")))
+                .close();
+
+        // test projection and filter
+        testStreamingRead(
+                        buildQuery(
+                                table,
+                                "from_currency, to_currency",
+                                "WHERE dt = '2022-01-02' AND from_currency = 'US Dollar'"),
+                        Arrays.asList(
+                                changelogRow("+I", "US Dollar", "Euro"),
+                                changelogRow("+I", "US Dollar", "Yen")))
+                .close();
+    }
+
+    @Test
+    public void testStreamingReadWriteSinglePartitionedRecordsWithMultiPk() throws Exception {
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // to_currency is USD
+                        changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
+                        changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+                        changelogRow("-D", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+                        changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
+                        changelogRow("+U", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+                        changelogRow("+U", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+                        // to_currency is Euro
+                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+                        changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+                        // to_currency is Yen
+                        changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+                        changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
+                        changelogRow("+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"),
+                        initialRecords,
+                        "dt:2022-01-01;dt:2022-01-02",
+                        false,
+                        "I,UA,D");
+
+        String table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"),
+                        false);
+
+        insertIntoFromTable(temporaryTable, table);
+
+        checkFileStorePath(table, Arrays.asList("dt=2022-01-01", "dt=2022-01-02"));
+
+        BlockingIterator<Row, Row> streamingItr =
+                testStreamingRead(
+                        buildSimpleQuery(table),
+                        Arrays.asList(
+                                changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+                                changelogRow(
+                                        "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+                                changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
+                                changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+                                changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+                                changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+                                changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+                                changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+                                changelogRow(
+                                        "+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-02")));
+
+        // test streaming consume changelog
+        insertIntoPartition(
+                table, "PARTITION (dt = '2022-01-03')", "('Chinese Yuan', 'HK Dollar', 1.231)");
+
+        validateStreamingReadResult(
+                streamingItr,
+                Collections.singletonList(
+                        changelogRow("+I", "Chinese Yuan", "HK Dollar", 1.231d, "2022-01-03")));
+        streamingItr.close();
+
+        // filter on partition and field filter
+        testStreamingRead(
+                        buildQuery(
+                                table,
+                                "*",
+                                "WHERE dt = '2022-01-02' AND from_currency = 'US Dollar'"),
+                        Collections.singletonList(
+                                changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02")))
+                .close();
+
+        // test projection and filter
+        testStreamingRead(
+                        buildQuery(
+                                table,
+                                "from_currency, to_currency",
+                                "WHERE dt = '2022-01-01' AND rate_by_to_currency IS NULL"),
+                        Collections.emptyList())
+                .close();
+    }
+
+    @Test
+    public void testStreamingReadWriteMultiPartitionedRecordsWithoutPk() throws Exception {
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // dt = 2022-01-01, hh = 10
+                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "10"),
+                        changelogRow("+I", "Euro", 116L, "2022-01-01", "10"),
+                        changelogRow("-D", "Euro", 116L, "2022-01-01", "10"),
+                        changelogRow("+I", "Yen", 1L, "2022-01-01", "10"),
+                        changelogRow("-D", "Yen", 1L, "2022-01-01", "10"),
+                        // dt = 2022-01-02, hh = 10
+                        changelogRow("+I", "Euro", 114L, "2022-01-02", "10"),
+                        changelogRow("-U", "Euro", 114L, "2022-01-02", "10"),
+                        changelogRow("+U", "Euro", 119L, "2022-01-02", "10"),
+                        changelogRow("-D", "Euro", 119L, "2022-01-02", "10"),
+                        changelogRow("+I", "Euro", 115L, "2022-01-02", "10"),
+                        // dt = 2022-01-02, hh = 11
+                        changelogRow("+I", "Yen", 1L, "2022-01-02", "11"));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"),
+                        initialRecords,
+                        "dt:2022-01-01,hh:10;dt:2022-01-02,hh:10;dt:2022-01-02,hh:11",
+                        false,
+                        "I,UA,UB,D");
+
+        String table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"),
+                        false);
+
+        insertIntoFromTable(temporaryTable, table);
+
+        checkFileStorePath(
+                table,
+                Arrays.asList("dt=2022-01-01,hh=10", "dt=2022-01-02,hh=10", "dt=2022-01-02,hh=11"));
+
+        BlockingIterator<Row, Row> streamingItr =
+                testStreamingRead(
+                        buildSimpleQuery(table),
+                        Arrays.asList(
+                                changelogRow("+I", "US Dollar", 102L, "2022-01-01", "10"),
+                                changelogRow("+I", "Euro", 115L, "2022-01-02", "10"),
+                                changelogRow("+I", "Yen", 1L, "2022-01-02", "11")));
+
+        // test streaming consume changelog
+        insertIntoPartition(table, "PARTITION (dt = '2022-04-02')", "('Euro', 116, '10')");
+
+        validateStreamingReadResult(
+                streamingItr,
+                Collections.singletonList(changelogRow("+I", "Euro", 116L, "2022-04-02", "10")));
+        streamingItr.close();
+
+        // filter on partition and field filter
+        testStreamingRead(
+                        buildQuery(table, "*", "WHERE dt = '2022-01-01' AND currency = 'Yen'"),
+                        Collections.emptyList())
+                .close();
+
+        // test projection and filter
+        testStreamingRead(
+                        buildQuery(table, "currency", "WHERE hh = '10' AND rate = 103"),
+                        Collections.emptyList())
+                .close();
+    }
+
+    // ----------------------------------------------------------------------------------------------------------------
+    // Latest
+    // ----------------------------------------------------------------------------------------------------------------
+    @Test
+    public void testReadLatestChangelogOfMultiPartitionedRecordsWithMultiPk() throws Exception {
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // to_currency is USD
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+                        changelogRow("+I", "Euro", "US Dollar", null, "2022-01-02", "20"),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"),
+                        changelogRow(
+                                "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"),
+                        changelogRow("+U", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
+                        changelogRow("-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+                        changelogRow("-D", "Euro", "US Dollar", null, "2022-01-02", "20"),
+                        changelogRow(
+                                "+U", "Singapore Dollar", "US Dollar", 0.76d, "2022-01-02", "20"),
+                        // to_currency is Euro
+                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"),
+                        changelogRow("+U", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
+                        // to_currency is Yen
+                        changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
+                        changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"),
+                        changelogRow("+U", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"),
+                        changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
+                        changelogRow("+U", "Singapore Dollar", "Yen", 90.1d, "2022-01-02", "20"));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        initialRecords,
+                        "dt:2022-01-02,hh:20;dt:2022-01-02,hh:21",
+                        false,
+                        "I,UA,D");
+
+        String table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        List<Row> expectedRecords = new ArrayList<>(initialRecords);
+        expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"));
+        expectedRecords.add(
+                changelogRow("-U", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"));
+        expectedRecords.add(
+                changelogRow("-U", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"));
+        expectedRecords.add(changelogRow("-U", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"));
+        expectedRecords.add(
+                changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"));
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+                        expectedRecords)
+                .close();
+
+        // test partition filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table, "*", "WHERE dt = '2022-01-02' AND hh = '21'", SCAN_LATEST),
+                        Collections.singletonList(
+                                changelogRow(
+                                        "+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")))
+                .close();
+
+        // test field filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table,
+                                "*",
+                                "WHERE rate_by_to_currency IS NOT NULL AND from_currency = 'US Dollar'",
+                                SCAN_LATEST),
+                        Arrays.asList(
+                                // to_currency is USD
+                                changelogRow(
+                                        "+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+                                changelogRow(
+                                        "-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+                                // to_currency is Euro
+                                changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+                                // to_currency is Yen
+                                changelogRow(
+                                        "+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")))
+                .close();
+
+        // test partition and field filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table,
+                                "*",
+                                "WHERE hh = '21' AND from_currency = 'US Dollar'",
+                                SCAN_LATEST),
+                        Collections.singletonList(
+                                changelogRow(
+                                        "+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")))
+                .close();
+
+        // test projection
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table, "from_currency, to_currency", "", SCAN_LATEST),
+                        Arrays.asList(
+                                // to_currency is USD
+                                changelogRow("+I", "US Dollar", "US Dollar"),
+                                changelogRow("+I", "Euro", "US Dollar"),
+                                changelogRow("+I", "HK Dollar", "US Dollar"),
+                                changelogRow("+I", "Yen", "US Dollar"),
+                                changelogRow("+I", "Singapore Dollar", "US Dollar"),
+                                changelogRow("-D", "US Dollar", "US Dollar"),
+                                changelogRow("-D", "Euro", "US Dollar"),
+                                // to_currency is Euro
+                                changelogRow("+I", "US Dollar", "Euro"),
+                                changelogRow("+I", "Singapore Dollar", "Euro"),
+                                // to_currency is Yen
+                                changelogRow("+I", "Yen", "Yen"),
+                                changelogRow("+I", "Chinese Yuan", "Yen"),
+                                changelogRow("+I", "Singapore Dollar", "Yen"),
+                                changelogRow("+I", "US Dollar", "Yen")))
+                .close();
+
+        // test projection and filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING",
+                                "hh STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table,
+                                "from_currency, to_currency",
+                                "WHERE rate_by_to_currency > 100",
+                                SCAN_LATEST),
+                        Collections.singletonList(changelogRow("+I", "US Dollar", "Yen")))
+                .close();
+    }
+
+    @Test
+    public void testReadLatestChangelogOfSinglePartitionedRecordsWithMultiPk() throws Exception {
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // to_currency is USD
+                        changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
+                        changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+                        changelogRow("-D", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+                        changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
+                        changelogRow("+U", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+                        changelogRow("+U", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+                        // to_currency is Euro
+                        changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+                        changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+                        // to_currency is Yen
+                        changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+                        changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
+                        changelogRow("+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"),
+                        initialRecords,
+                        "dt:2022-01-01;dt:2022-01-02",
+                        false,
+                        "I,UA,D");
+
+        String table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"),
+                        true);
+
+        List<Row> expectedRecords = new ArrayList<>(initialRecords);
+        expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"));
+        expectedRecords.add(changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01"));
+        expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"));
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+                        expectedRecords)
+                .close();
+
+        // test partition filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table, "*", "WHERE dt = '2022-01-02'", SCAN_LATEST),
+                        Arrays.asList(
+                                changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+                                changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+                                changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+                                changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+                                changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+                                changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
+                                changelogRow(
+                                        "+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"),
+                                changelogRow(
+                                        "-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02")))
+                .close();
+
+        // test field filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table, "*", "WHERE rate_by_to_currency IS NULL", SCAN_LATEST),
+                        Arrays.asList(
+                                changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
+                                changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
+                                changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"),
+                                changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01")))
+                .close();
+
+        // test partition and field filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table,
+                                "*",
+                                "WHERE dt = '2022-01-02' AND from_currency = 'Yen'",
+                                SCAN_LATEST),
+                        Collections.singletonList(
+                                changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02")))
+                .close();
+
+        // test projection and filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE",
+                                "dt STRING"),
+                        Arrays.asList("from_currency", "to_currency", "dt"),
+                        Collections.singletonList("dt"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table,
+                                "from_currency, to_currency",
+                                "WHERE rate_by_to_currency > 100",
+                                SCAN_LATEST),
+                        Collections.singletonList(changelogRow("+U", "Singapore Dollar", "Yen")))
+                .close();
+    }
+
+    @Test
+    public void testReadLatestChangelogOfNonPartitionedRecordsWithMultiPk() throws Exception {
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // to_currency is USD
+                        changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
+                        changelogRow("+I", "Euro", "US Dollar", 1.11d),
+                        changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+                        changelogRow("+U", "Euro", "US Dollar", 1.12d),
+                        changelogRow("+I", "Yen", "US Dollar", 0.0082d),
+                        changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+                        changelogRow("+U", "Yen", "US Dollar", 0.0081d),
+                        changelogRow("-D", "US Dollar", "US Dollar", 1.0d),
+                        changelogRow("-D", "Yen", "US Dollar", 0.0081d),
+                        // to_currency is Euro
+                        changelogRow("+I", "US Dollar", "Euro", 0.9d),
+                        changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+                        changelogRow("+U", "Singapore Dollar", "Euro", 0.69d),
+                        // to_currency is Yen
+                        changelogRow("+I", "Yen", "Yen", 1.0d),
+                        changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+                        changelogRow("+I", "Singapore Dollar", "Yen", 90.32d),
+                        changelogRow("-D", "Yen", "Yen", 1.0d),
+                        changelogRow("+U", "Singapore Dollar", "Yen", 122.46d),
+                        changelogRow("+U", "Singapore Dollar", "Yen", 122d));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE"),
+                        Arrays.asList("from_currency", "to_currency"),
+                        Collections.emptyList(),
+                        initialRecords,
+                        null,
+                        false,
+                        "I,UA,D");
+
+        String table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE"),
+                        Arrays.asList("from_currency", "to_currency"),
+                        Collections.emptyList(),
+                        true);
+
+        List<Row> expectedRecords = new ArrayList<>(initialRecords);
+        expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d));
+        expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", 1.11d));
+        expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Euro", 0.67d));
+        expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d));
+        expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 122.46d));
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+                        expectedRecords)
+                .close();
+
+        // test field filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE"),
+                        Arrays.asList("from_currency", "to_currency"),
+                        Collections.emptyList(),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table,
+                                "*",
+                                "WHERE rate_by_to_currency < 1 OR rate_by_to_currency > 100",
+                                SCAN_LATEST),
+                        Arrays.asList(
+                                // to_currency is USD
+                                changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+                                changelogRow("+I", "Yen", "US Dollar", 0.0082d),
+                                changelogRow("-U", "Yen", "US Dollar", 0.0082d),
+                                changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+                                changelogRow("+U", "Yen", "US Dollar", 0.0081d),
+                                changelogRow("-D", "Yen", "US Dollar", 0.0081d),
+                                // to_currency is Euro
+                                changelogRow("+I", "US Dollar", "Euro", 0.9d),
+                                changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+                                changelogRow("-U", "Singapore Dollar", "Euro", 0.67d),
+                                changelogRow("+U", "Singapore Dollar", "Euro", 0.69d),
+                                // to_currency is Yen
+                                changelogRow("+U", "Singapore Dollar", "Yen", 122.46d),
+                                changelogRow("-U", "Singapore Dollar", "Yen", 122.46d),
+                                changelogRow("+U", "Singapore Dollar", "Yen", 122d)))
+                .close();
+
+        // test projection and filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList(
+                                "from_currency STRING",
+                                "to_currency STRING",
+                                "rate_by_to_currency DOUBLE"),
+                        Arrays.asList("from_currency", "to_currency"),
+                        Collections.emptyList(),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table,
+                                "from_currency, to_currency",
+                                "WHERE rate_by_to_currency < 1 OR rate_by_to_currency > 100",
+                                SCAN_LATEST),
+                        Arrays.asList(
+                                // to_currency is USD
+                                changelogRow("+I", "HK Dollar", "US Dollar"),
+                                changelogRow("+I", "Yen", "US Dollar"),
+                                changelogRow("-U", "Yen", "US Dollar"),
+                                changelogRow("+I", "Singapore Dollar", "US Dollar"),
+                                changelogRow("+U", "Yen", "US Dollar"),
+                                changelogRow("-D", "Yen", "US Dollar"),
+                                // to_currency is Euro
+                                changelogRow("+I", "US Dollar", "Euro"),
+                                changelogRow("+I", "Singapore Dollar", "Euro"),
+                                changelogRow("-U", "Singapore Dollar", "Euro"),
+                                changelogRow("+U", "Singapore Dollar", "Euro"),
+                                // to_currency is Yen
+                                changelogRow("+U", "Singapore Dollar", "Yen"),
+                                changelogRow("-U", "Singapore Dollar", "Yen"),
+                                changelogRow("+U", "Singapore Dollar", "Yen")))
+                .close();
+    }
+
+    @Test
+    public void testReadLatestChangelogOfMultiPartitionedRecordsWithOnePk() throws Exception {
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // dt = 2022-01-01, hh = "15"
+                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+                        changelogRow("+I", "Euro", 114L, "2022-01-01", "15"),
+                        changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+                        changelogRow("+U", "Euro", 116L, "2022-01-01", "15"),
+                        changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+                        changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+                        // dt = 2022-01-02, hh = "23"
+                        changelogRow("+I", "Euro", 119L, "2022-01-02", "23"),
+                        changelogRow("+U", "Euro", 119L, "2022-01-02", "23"));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Arrays.asList("currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        initialRecords,
+                        "dt:2022-01-01,hh:15;dt:2022-01-02,hh:23",
+                        false,
+                        "I,UA,D");
+
+        String table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Arrays.asList("currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        List<Row> expectedRecords = new ArrayList<>(initialRecords);
+        expectedRecords.remove(changelogRow("+U", "Euro", 119L, "2022-01-02", "23"));
+        expectedRecords.add(changelogRow("-U", "Euro", 114L, "2022-01-01", "15"));
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+                        expectedRecords)
+                .close();
+
+        // test partition filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Arrays.asList("currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table, "*", "WHERE dt >= '2022-01-02'", SCAN_LATEST),
+                        Collections.singletonList(
+                                changelogRow("+I", "Euro", 119L, "2022-01-02", "23")))
+                .close();
+
+        // test field filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Arrays.asList("currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "WHERE rate = 1", SCAN_LATEST),
+                        Arrays.asList(
+                                changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+                                changelogRow("-D", "Yen", 1L, "2022-01-01", "15")))
+                .close();
+
+        // test projection and filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Arrays.asList("currency", "dt", "hh"),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table, "currency", "WHERE rate = 1", SCAN_LATEST),
+                        Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")))
+                .close();
+    }
+
+    @Test
+    public void testReadLatestChangelogOfMultiPartitionedRecordsWithoutPk() throws Exception {
+        List<Row> initialRecords =
+                Arrays.asList(
+                        // dt = 2022-01-01, hh = 15
+                        changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+                        changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+                        changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+                        changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+                        changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+                        // dt = 2022-01-02, hh = 20
+                        changelogRow("+I", "Euro", 114L, "2022-01-02", "20"),
+                        changelogRow("-U", "Euro", 114L, "2022-01-02", "20"),
+                        changelogRow("+U", "Euro", 119L, "2022-01-02", "20"),
+                        changelogRow("-D", "Euro", 119L, "2022-01-02", "20"),
+                        changelogRow("+I", "Euro", 115L, "2022-01-02", "20"));
+
+        String temporaryTable =
+                createTemporaryTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"),
+                        initialRecords,
+                        "dt:2022-01-01,hh:15;dt:2022-01-02,hh:20",
+                        false,
+                        "I,UA,UB,D");
+
+        String table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+                        Arrays.asList(
+                                // dt = 2022-01-01, hh = 15
+                                changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+                                changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+                                changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+                                changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+                                changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+                                // dt = 2022-01-02, hh = 20
+                                changelogRow("+I", "Euro", 114L, "2022-01-02", "20"),
+                                changelogRow("-D", "Euro", 114L, "2022-01-02", "20"),
+                                changelogRow("+I", "Euro", 119L, "2022-01-02", "20"),
+                                changelogRow("-D", "Euro", 119L, "2022-01-02", "20"),
+                                changelogRow("+I", "Euro", 115L, "2022-01-02", "20")))
+                .close();
+
+        // test partition filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "WHERE hh <> '20'", SCAN_LATEST),
+                        Arrays.asList(
+                                changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+                                changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+                                changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+                                changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+                                changelogRow("-D", "Yen", 1L, "2022-01-01", "15")))
+                .close();
+
+        // test field filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "WHERE hh <> '20'", SCAN_LATEST),
+                        Arrays.asList(
+                                changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+                                changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+                                changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+                                changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+                                changelogRow("-D", "Yen", 1L, "2022-01-01", "15")))
+                .close();
+
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(table, "*", "WHERE rate = 1", SCAN_LATEST),
+                        Arrays.asList(
+                                changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+                                changelogRow("-D", "Yen", 1L, "2022-01-01", "15")))
+                .close();
+
+        // test projection and filter
+        table =
+                createTableWithKafkaLog(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+                        Collections.emptyList(),
+                        Arrays.asList("dt", "hh"),
+                        true);
+
+        testStreamingReadWithReadFirst(
+                        temporaryTable,
+                        table,
+                        buildQueryWithTableOptions(
+                                table, "currency", "WHERE rate = 1", SCAN_LATEST),
+                        Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")))
+                .close();
+    }
+}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
index 00d0a3f9..155898d3 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
@@ -24,22 +24,16 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.EnvironmentSettings;
 import org.apache.flink.table.api.Schema;
 import org.apache.flink.table.api.TableDescriptor;
-import org.apache.flink.table.api.TableResult;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.table.catalog.CatalogBaseTable;
 import org.apache.flink.table.catalog.ObjectIdentifier;
 import org.apache.flink.table.catalog.ObjectPath;
 import org.apache.flink.table.catalog.ResolvedCatalogTable;
-import org.apache.flink.table.store.CoreOptions.StartupMode;
 import org.apache.flink.table.store.file.utils.BlockingIterator;
 import org.apache.flink.table.store.kafka.KafkaTableTestBase;
 import org.apache.flink.types.Row;
 
 import javax.annotation.Nullable;
 
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -47,18 +41,13 @@ import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
 
-import static org.apache.flink.table.store.CoreOptions.SCAN_MODE;
 import static org.apache.flink.table.store.connector.FlinkConnectorOptions.LOG_SYSTEM;
 import static org.apache.flink.table.store.connector.FlinkConnectorOptions.ROOT_PATH;
-import static org.apache.flink.table.store.connector.FlinkConnectorOptions.relativeTablePath;
 import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.prepareHelperSourceWithChangelogRecords;
 import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.prepareHelperSourceWithInsertOnlyRecords;
 import static org.apache.flink.table.store.connector.ShowCreateUtil.buildInsertIntoQuery;
-import static org.apache.flink.table.store.connector.ShowCreateUtil.buildInsertOverwriteQuery;
 import static org.apache.flink.table.store.connector.ShowCreateUtil.buildSelectQuery;
-import static org.apache.flink.table.store.connector.ShowCreateUtil.buildSimpleSelectQuery;
 import static org.apache.flink.table.store.connector.ShowCreateUtil.createTableLikeDDL;
 import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -69,71 +58,6 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
 
     // ------------------------ Tools ----------------------------------
 
-    protected void checkFileStorePath(
-            StreamTableEnvironment tEnv, String managedTable, @Nullable String partitionList) {
-        String relativeFilePath =
-                relativeTablePath(
-                        ObjectIdentifier.of(
-                                tEnv.getCurrentCatalog(), tEnv.getCurrentDatabase(), managedTable));
-        // check snapshot file path
-        assertThat(Paths.get(rootPath, relativeFilePath, "snapshot")).exists();
-        // check manifest file path
-        assertThat(Paths.get(rootPath, relativeFilePath, "manifest")).exists();
-        // check data file path
-        if (partitionList == null) {
-            // at least exists bucket-0
-            assertThat(Paths.get(rootPath, relativeFilePath, "bucket-0")).exists();
-        } else {
-            Arrays.stream(partitionList.split(";"))
-                    .map(str -> str.replaceAll(":", "="))
-                    .map(str -> str.replaceAll(",", "/"))
-                    .map(str -> str.replaceAll("null", "__DEFAULT_PARTITION__"))
-                    .collect(Collectors.toList())
-                    .forEach(
-                            partition -> {
-                                assertThat(Paths.get(rootPath, relativeFilePath, partition))
-                                        .exists();
-                                assertThat(
-                                                Paths.get(
-                                                        rootPath,
-                                                        relativeFilePath,
-                                                        partition,
-                                                        "bucket-0"))
-                                        .exists();
-                            });
-        }
-    }
-
-    protected static BlockingIterator<Row, Row> collectAndCheck(
-            StreamTableEnvironment tEnv,
-            String managedTable,
-            Map<String, String> hints,
-            @Nullable String filter,
-            List<Row> expectedRecords)
-            throws Exception {
-        List<Row> actual = new ArrayList<>();
-        BlockingIterator<Row, Row> iterator =
-                collect(
-                        tEnv,
-                        filter == null
-                                ? buildSimpleSelectQuery(managedTable, hints)
-                                : buildSelectQuery(
-                                        managedTable, hints, filter, Collections.emptyList()),
-                        expectedRecords.size(),
-                        actual);
-        assertThat(actual).containsExactlyInAnyOrderElementsOf(expectedRecords);
-        return iterator;
-    }
-
-    protected static BlockingIterator<Row, Row> collect(
-            StreamTableEnvironment tEnv, String selectQuery, int expectedSize, List<Row> actual)
-            throws Exception {
-        TableResult result = tEnv.executeSql(selectQuery);
-        BlockingIterator<Row, Row> iterator = BlockingIterator.of(result.collect());
-        actual.addAll(iterator.collect(expectedSize));
-        return iterator;
-    }
-
     protected static void assertNoMoreRecords(BlockingIterator<Row, Row> iterator)
             throws Exception {
         List<Row> expectedRecords = Collections.emptyList();
@@ -148,107 +72,6 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
         assertThat(expectedRecords).isEmpty();
     }
 
-    protected String collectAndCheckBatchReadWrite(
-            List<String> partitions,
-            List<String> primaryKeys,
-            @Nullable String filter,
-            List<String> projection,
-            List<Row> expected)
-            throws Exception {
-        return collectAndCheckUnderSameEnv(
-                        false,
-                        false,
-                        true,
-                        partitions,
-                        primaryKeys,
-                        Collections.emptyList(),
-                        null,
-                        true,
-                        Collections.emptyMap(),
-                        filter,
-                        projection,
-                        expected)
-                .f0;
-    }
-
-    protected String collectAndCheckStreamingReadWriteWithClose(
-            boolean enableLogStore,
-            List<String> partitions,
-            List<String> primaryKeys,
-            Map<String, String> readHints,
-            @Nullable String filter,
-            List<String> projection,
-            List<Row> expected)
-            throws Exception {
-        Tuple2<String, BlockingIterator<Row, Row>> tuple =
-                collectAndCheckUnderSameEnv(
-                        true,
-                        enableLogStore,
-                        false,
-                        partitions,
-                        primaryKeys,
-                        Collections.emptyList(),
-                        null,
-                        true,
-                        readHints,
-                        filter,
-                        projection,
-                        expected);
-        tuple.f1.close();
-        return tuple.f0;
-    }
-
-    protected Tuple2<String, BlockingIterator<Row, Row>>
-            collectAndCheckStreamingReadWriteWithoutClose(
-                    List<String> partitions,
-                    List<String> primaryKeys,
-                    Map<String, String> readHints,
-                    @Nullable String filter,
-                    List<String> projection,
-                    List<Row> expected)
-                    throws Exception {
-        return collectAndCheckUnderSameEnv(
-                true,
-                true,
-                false,
-                partitions,
-                primaryKeys,
-                Collections.emptyList(),
-                null,
-                true,
-                readHints,
-                filter,
-                projection,
-                expected);
-    }
-
-    protected void collectLatestLogAndCheck(
-            boolean insertOnly,
-            List<String> partitionKeys,
-            List<String> primaryKeys,
-            @Nullable String filter,
-            List<String> projection,
-            List<Row> expected)
-            throws Exception {
-        Map<String, String> hints = new HashMap<>();
-        hints.put(SCAN_MODE.key(), StartupMode.LATEST.name().toLowerCase());
-        collectAndCheckUnderSameEnv(
-                        true,
-                        true,
-                        insertOnly,
-                        partitionKeys,
-                        primaryKeys,
-                        Collections.emptyList(),
-                        null,
-                        false,
-                        hints,
-                        filter,
-                        projection,
-                        expected)
-                .f1
-                .close();
-    }
-
     protected Tuple2<String, String> createSourceAndManagedTable(
             boolean streaming,
             boolean enableLogStore,
@@ -366,32 +189,6 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
         return Tuple2.of(managedTable, iterator);
     }
 
-    protected void prepareEnvAndOverwrite(
-            String managedTable,
-            Map<String, String> staticPartitions,
-            List<String[]> overwriteRecords)
-            throws Exception {
-        prepareEnvAndOverwrite(
-                managedTable,
-                buildInsertOverwriteQuery(managedTable, staticPartitions, overwriteRecords));
-    }
-
-    protected void prepareEnvAndOverwrite(String managedTable, String query) throws Exception {
-        final StreamTableEnvironment batchEnv =
-                StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
-        registerTable(batchEnv, managedTable);
-        batchEnv.executeSql(query).await();
-    }
-
-    protected void registerTable(StreamTableEnvironment tEnvToRegister, String managedTable)
-            throws Exception {
-        String cat = tEnv.getCurrentCatalog();
-        String db = tEnv.getCurrentDatabase();
-        ObjectPath objectPath = new ObjectPath(db, managedTable);
-        CatalogBaseTable table = tEnv.getCatalog(cat).get().getTable(objectPath);
-        tEnvToRegister.getCatalog(cat).get().createTable(objectPath, table, false);
-    }
-
     protected static StreamExecutionEnvironment buildStreamEnv() {
         return buildStreamEnv(2);
     }
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
index 1ddb1455..355bfa80 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
@@ -37,7 +37,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
 /**
  * SHOW CREATE statement Util.
@@ -67,41 +66,6 @@ public class ShowCreateUtil {
         return ddl.toString();
     }
 
-    public static String buildInsertOverwriteQuery(
-            String managedTableName,
-            Map<String, String> staticPartitions,
-            List<String[]> overwriteRecords) {
-        StringBuilder insertDmlBuilder =
-                new StringBuilder(String.format("INSERT OVERWRITE `%s`", managedTableName));
-        if (staticPartitions.size() > 0) {
-            String partitionString =
-                    staticPartitions.entrySet().stream()
-                            .map(
-                                    entry ->
-                                            String.format(
-                                                    "%s = %s", entry.getKey(), entry.getValue()))
-                            .collect(Collectors.joining(", "));
-            insertDmlBuilder.append(String.format("PARTITION (%s)", partitionString));
-        }
-        insertDmlBuilder.append("\n VALUES ");
-        overwriteRecords.forEach(
-                record -> {
-                    int arity = record.length;
-                    insertDmlBuilder.append("(");
-                    IntStream.range(0, arity)
-                            .forEach(i -> insertDmlBuilder.append(record[i]).append(", "));
-
-                    if (arity > 0) {
-                        int len = insertDmlBuilder.length();
-                        insertDmlBuilder.delete(len - 2, len);
-                    }
-                    insertDmlBuilder.append("), ");
-                });
-        int len = insertDmlBuilder.length();
-        insertDmlBuilder.delete(len - 2, len);
-        return insertDmlBuilder.toString();
-    }
-
     public static String buildInsertIntoQuery(String sourceTableName, String managedTableName) {
         return buildInsertIntoQuery(
                 sourceTableName, managedTableName, Collections.emptyMap(), Collections.emptyMap());
@@ -132,10 +96,6 @@ public class ShowCreateUtil {
         return insertDmlBuilder.toString();
     }
 
-    public static String buildSimpleSelectQuery(String tableName, Map<String, String> hints) {
-        return buildSelectQuery(tableName, hints, null, Collections.emptyList());
-    }
-
     public static String buildSelectQuery(
             String tableName,
             Map<String, String> hints,
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
index 677c15eb..6327ed58 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
@@ -19,13 +19,10 @@
 package org.apache.flink.table.store.connector;
 
 import org.apache.flink.table.store.CoreOptions;
-import org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil;
 import org.apache.flink.table.store.file.utils.BlockingIterator;
 import org.apache.flink.table.store.kafka.KafkaTableTestBase;
 import org.apache.flink.types.Row;
 
-import org.apache.kafka.clients.admin.AdminClient;
-import org.apache.kafka.clients.admin.NewTopic;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -35,19 +32,17 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.UUID;
 
 import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
 import static org.apache.flink.table.store.CoreOptions.SCAN_MODE;
 import static org.apache.flink.table.store.CoreOptions.SCAN_TIMESTAMP_MILLIS;
-import static org.apache.flink.table.store.connector.FlinkConnectorOptions.LOG_SYSTEM;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.SCAN_LATEST;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.assertNoMoreRecords;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQuery;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQueryWithTableOptions;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildSimpleQuery;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.checkFileStorePath;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTableWithKafkaLog;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTemporaryTable;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.init;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertInto;
@@ -57,20 +52,11 @@ import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testBatchRead;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingRead;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingReadWithReadFirst;
-import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
-import static org.apache.flink.table.store.kafka.KafkaLogOptions.TOPIC;
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.validateStreamingReadResult;
 
 /** Streaming reading and writing with Kafka log IT cases. */
 public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBase {
 
-    private final Map<String, String> scanLatest =
-            new HashMap<String, String>() {
-                {
-                    put(SCAN_MODE.key(), CoreOptions.StartupMode.LATEST.toString());
-                }
-            };
-
     @Before
     public void setUp() throws Exception {
         init(createAndRegisterTempFile("").toString());
@@ -107,10 +93,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
-                        Collections.singletonList("dt"));
+                        Collections.singletonList("dt"),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -132,11 +119,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         insertIntoPartition(table, "PARTITION (dt = '2022-01-04')", "('Yen', 20)");
 
-        assertThat(streamItr.collect(2))
-                .containsExactlyInAnyOrderElementsOf(
-                        Arrays.asList(
-                                changelogRow("+I", "HK Dollar", 100L, "2022-01-03"),
-                                changelogRow("+I", "Yen", 20L, "2022-01-03")));
+        validateStreamingReadResult(
+                streamItr,
+                Arrays.asList(
+                        changelogRow("+I", "HK Dollar", 100L, "2022-01-03"),
+                        changelogRow("+I", "Yen", 20L, "2022-01-03")));
 
         // overwrite partition 2022-01-02
         insertOverwritePartition(
@@ -216,10 +203,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,UB,D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Collections.emptyList(),
-                        Collections.singletonList("dt"));
+                        Collections.singletonList("dt"),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -298,10 +286,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I, UA, D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
-                        Collections.emptyList());
+                        Collections.emptyList(),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -360,10 +349,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,UB,D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
-                        Collections.emptyList());
+                        Collections.emptyList(),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -433,7 +423,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
                         Collections.singletonList("dt"),
@@ -443,7 +433,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                 testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "*", "", scanLatest),
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
                                 changelogRow("+I", "Euro", 114L, "2022-01-01"),
@@ -456,20 +446,20 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test only read the latest log
         insertInto(table, "('US Dollar', 104, '2022-01-01')", "('Euro', 100, '2022-01-02')");
-        assertThat(streamItr.collect(4))
-                .containsExactlyInAnyOrderElementsOf(
-                        Arrays.asList(
-                                changelogRow("-U", "US Dollar", 102L, "2022-01-01"),
-                                changelogRow("+U", "US Dollar", 104L, "2022-01-01"),
-                                changelogRow("-U", "Euro", 119L, "2022-01-02"),
-                                changelogRow("+U", "Euro", 100L, "2022-01-02")));
+        validateStreamingReadResult(
+                streamItr,
+                Arrays.asList(
+                        changelogRow("-U", "US Dollar", 102L, "2022-01-01"),
+                        changelogRow("+U", "US Dollar", 104L, "2022-01-01"),
+                        changelogRow("-U", "Euro", 119L, "2022-01-02"),
+                        changelogRow("+U", "Euro", 100L, "2022-01-02")));
 
         assertNoMoreRecords(streamItr);
         streamItr.close();
 
         // test partition filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
                         Collections.singletonList("dt"),
@@ -479,7 +469,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "*", "WHERE dt = '2022-01-01'", scanLatest),
+                                table, "*", "WHERE dt = '2022-01-01'", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
                                 changelogRow("+I", "Euro", 114L, "2022-01-01"),
@@ -492,7 +482,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test field filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
                         Collections.singletonList("dt"),
@@ -502,14 +492,14 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "*", "WHERE currency = 'Yen'", scanLatest),
+                                table, "*", "WHERE currency = 'Yen'", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "Yen", 1L, "2022-01-01"),
                                 changelogRow("-D", "Yen", 1L, "2022-01-01")))
                 .close();
 
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
                         Collections.singletonList("dt"),
@@ -518,7 +508,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "*", "WHERE rate = 114", scanLatest),
+                        buildQueryWithTableOptions(table, "*", "WHERE rate = 114", SCAN_LATEST),
                         Arrays.asList(
                                 // part = 2022-01-01
                                 changelogRow("+I", "Euro", 114L, "2022-01-01"),
@@ -527,7 +517,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test partition and field filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
                         Collections.singletonList("dt"),
@@ -537,13 +527,13 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "*", "WHERE rate = 114 AND dt = '2022-01-02'", scanLatest),
+                                table, "*", "WHERE rate = 114 AND dt = '2022-01-02'", SCAN_LATEST),
                         Collections.emptyList())
                 .close();
 
         // test projection
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
                         Collections.singletonList("dt"),
@@ -552,7 +542,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "rate", "", scanLatest),
+                        buildQueryWithTableOptions(table, "rate", "", SCAN_LATEST),
                         Arrays.asList(
                                 // part = 2022-01-01
                                 changelogRow("+I", 102L), // US Dollar
@@ -569,7 +559,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test projection and filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
                         Collections.singletonList("dt"),
@@ -579,7 +569,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "rate", "WHERE dt = '2022-01-02'", scanLatest),
+                                table, "rate", "WHERE dt = '2022-01-02'", SCAN_LATEST),
                         Collections.singletonList(changelogRow("+I", 119L)) // Euro
                         )
                 .close();
@@ -613,7 +603,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,UB,D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Collections.emptyList(),
                         Collections.singletonList("dt"),
@@ -622,7 +612,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "*", "", scanLatest),
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
                         Arrays.asList(
                                 // part = 2022-01-01
                                 changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
@@ -640,7 +630,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test partition filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Collections.emptyList(),
                         Collections.singletonList("dt"),
@@ -650,7 +640,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "*", "WHERE dt = '2022-01-01'", scanLatest),
+                                table, "*", "WHERE dt = '2022-01-01'", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
                                 changelogRow("+I", "Euro", 116L, "2022-01-01"),
@@ -664,7 +654,10 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "*", "WHERE currency = 'US Dollar' OR rate = 1", scanLatest),
+                                table,
+                                "*",
+                                "WHERE currency = 'US Dollar' OR rate = 1",
+                                SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
                                 changelogRow("+I", "Yen", 1L, "2022-01-01"),
@@ -673,7 +666,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test partition and field filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Collections.emptyList(),
                         Collections.singletonList("dt"),
@@ -683,7 +676,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "*", "WHERE dt = '2022-01-02' AND rate = 114", scanLatest),
+                                table, "*", "WHERE dt = '2022-01-02' AND rate = 114", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "Euro", 114L, "2022-01-02"),
                                 changelogRow("-D", "Euro", 114L, "2022-01-02")))
@@ -691,7 +684,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test projection
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Collections.emptyList(),
                         Collections.singletonList("dt"),
@@ -700,7 +693,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "rate", "", scanLatest),
+                        buildQueryWithTableOptions(table, "rate", "", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", 102L),
                                 changelogRow("+I", 116L),
@@ -716,7 +709,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test projection and filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Collections.emptyList(),
                         Collections.singletonList("dt"),
@@ -726,7 +719,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "rate", "WHERE dt <> '2022-01-01'", scanLatest),
+                                table, "rate", "WHERE dt <> '2022-01-01'", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", 114L),
                                 changelogRow("-D", 114L),
@@ -760,7 +753,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
                         Collections.emptyList(),
@@ -769,7 +762,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "*", "", scanLatest),
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar", 102L),
                                 changelogRow("+I", "Euro", 114L),
@@ -783,7 +776,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test field filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
                         Collections.emptyList(),
@@ -793,7 +786,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "*", "WHERE currency = 'Euro'", scanLatest),
+                                table, "*", "WHERE currency = 'Euro'", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "Euro", 114L),
                                 changelogRow("-U", "Euro", 114L),
@@ -804,7 +797,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test projection
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
                         Collections.emptyList(),
@@ -813,7 +806,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "currency", "", scanLatest),
+                        buildQueryWithTableOptions(table, "currency", "", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar"),
                                 changelogRow("+I", "Euro"),
@@ -825,7 +818,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test projection and filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
                         Collections.emptyList(),
@@ -835,7 +828,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "rate", "WHERE currency = 'Euro'", scanLatest),
+                                table, "rate", "WHERE currency = 'Euro'", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", 114L),
                                 changelogRow("-U", 114L),
@@ -873,7 +866,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,UB,D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
                         Collections.emptyList(),
@@ -882,7 +875,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "*", "", scanLatest),
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar", 102L),
                                 changelogRow("+I", "Euro", 114L),
@@ -900,7 +893,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test field filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
                         Collections.emptyList(),
@@ -910,7 +903,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "*", "WHERE currency = 'Euro'", scanLatest),
+                                table, "*", "WHERE currency = 'Euro'", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "Euro", 114L),
                                 changelogRow("-D", "Euro", 114L),
@@ -923,7 +916,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test projection
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
                         Collections.emptyList(),
@@ -932,7 +925,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "currency, rate", "", scanLatest),
+                        buildQueryWithTableOptions(table, "currency, rate", "", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar", 102L),
                                 changelogRow("+I", "Euro", 114L),
@@ -950,7 +943,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test projection and filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
                         Collections.emptyList(),
@@ -960,7 +953,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "currency", "WHERE currency IS NOT NULL", scanLatest),
+                                table, "currency", "WHERE currency IS NOT NULL", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar"),
                                 changelogRow("+I", "Euro"),
@@ -976,7 +969,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                 .close();
 
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
                         Collections.emptyList(),
@@ -986,7 +979,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "currency", "WHERE rate = 119", scanLatest),
+                                table, "currency", "WHERE rate = 119", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "Euro"),
                                 changelogRow("-D", "Euro"),
@@ -1016,7 +1009,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
                         Collections.emptyList(),
@@ -1025,7 +1018,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "*", "", scanLatest),
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
                         initialRecords)
                 .close();
 
@@ -1041,7 +1034,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I");
 
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
                         Collections.emptyList(),
@@ -1050,7 +1043,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "*", "", scanLatest),
+                        buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "US Dollar", 102L),
                                 changelogRow("+I", "Euro", 114L),
@@ -1061,7 +1054,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test field filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
                         Collections.emptyList(),
@@ -1070,14 +1063,14 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "*", "WHERE rate = 114", scanLatest),
+                        buildQueryWithTableOptions(table, "*", "WHERE rate = 114", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", "Euro", 114L), changelogRow("-U", "Euro", 114L)))
                 .close();
 
         // test projection
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
                         Collections.emptyList(),
@@ -1086,7 +1079,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
         testStreamingReadWithReadFirst(
                         temporaryTable,
                         table,
-                        buildQueryWithTableOptions(table, "rate", "", scanLatest),
+                        buildQueryWithTableOptions(table, "rate", "", SCAN_LATEST),
                         Arrays.asList(
                                 changelogRow("+I", 102L),
                                 changelogRow("+I", 114L),
@@ -1097,7 +1090,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // test projection and filter
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
                         Collections.emptyList(),
@@ -1107,7 +1100,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         temporaryTable,
                         table,
                         buildQueryWithTableOptions(
-                                table, "currency", "WHERE rate = 114", scanLatest),
+                                table, "currency", "WHERE rate = 114", SCAN_LATEST),
                         Arrays.asList(changelogRow("+I", "Euro"), changelogRow("-U", "Euro")))
                 .close();
     }
@@ -1142,10 +1135,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Collections.emptyList(),
-                        Collections.singletonList("dt"));
+                        Collections.singletonList("dt"),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -1166,10 +1160,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I");
 
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
-                        Collections.singletonList("dt"));
+                        Collections.singletonList("dt"),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -1205,10 +1200,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I");
 
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.singletonList("currency"),
-                        Collections.emptyList());
+                        Collections.emptyList(),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -1234,10 +1230,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I");
 
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
-                        Collections.emptyList());
+                        Collections.emptyList(),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -1277,10 +1274,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT"),
                         Collections.emptyList(),
-                        Collections.emptyList());
+                        Collections.emptyList(),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -1321,10 +1319,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,UB,D");
 
         String table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Collections.emptyList(),
-                        Collections.singletonList("dt"));
+                        Collections.singletonList("dt"),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -1370,10 +1369,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         "I,UA,D");
 
         table =
-                createTable(
+                createTableWithKafkaLog(
                         Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
                         Arrays.asList("currency", "dt"),
-                        Collections.singletonList("dt"));
+                        Collections.singletonList("dt"),
+                        false);
 
         insertIntoFromTable(temporaryTable, table);
 
@@ -1395,50 +1395,6 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
     // Tools
     // ----------------------------------------------------------------------------------------------------------------
 
-    private String createTable(
-            List<String> fieldsSpec, List<String> primaryKeys, List<String> partitionKeys)
-            throws Exception {
-        return createTable(fieldsSpec, primaryKeys, partitionKeys, false);
-    }
-
-    private String createTable(
-            List<String> fieldsSpec,
-            List<String> primaryKeys,
-            List<String> partitionKeys,
-            boolean manuallyCreateLogTable)
-            throws Exception {
-        String topic = "topic_" + UUID.randomUUID();
-        String table =
-                ReadWriteTableTestUtil.createTable(
-                        fieldsSpec,
-                        primaryKeys,
-                        partitionKeys,
-                        new HashMap<String, String>() {
-                            {
-                                put(LOG_SYSTEM.key(), "kafka");
-                                put(BOOTSTRAP_SERVERS.key(), getBootstrapServers());
-                                put(TOPIC.key(), topic);
-                            }
-                        });
-
-        if (manuallyCreateLogTable) {
-            createKafkaLogTable(topic);
-        }
-
-        return table;
-    }
-
-    private void createKafkaLogTable(String topic) throws Exception {
-        Properties properties = new Properties();
-        properties.put("bootstrap.servers", getBootstrapServers());
-
-        try (AdminClient adminClient = AdminClient.create(properties)) {
-            NewTopic topicObj =
-                    new NewTopic(topic, Optional.of(1), Optional.empty()).configs(new HashMap<>());
-            adminClient.createTopics(Collections.singleton(topicObj)).all().get();
-        }
-    }
-
     private Map<String, String> scanFromTimeStampMillis(Long timeStampMillis) {
         return new HashMap<String, String>() {
             {
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
index 22279f28..b5b5f592 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
@@ -32,11 +32,13 @@ import org.apache.flink.table.store.table.source.snapshot.SnapshotEnumerator;
 import org.apache.flink.table.store.types.DataType;
 import org.apache.flink.table.store.types.DataTypes;
 import org.apache.flink.table.store.types.RowType;
+import org.apache.flink.table.store.utils.CommonTestUtils;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -178,9 +180,13 @@ public class CompactActionITCase extends ActionITCaseBase {
                 60_000);
 
         // assert dedicated compact job will expire snapshots
-        Assertions.assertEquals(
-                snapshotManager.latestSnapshotId() - 2,
-                (long) snapshotManager.earliestSnapshotId());
+        CommonTestUtils.waitUtil(
+                () ->
+                        snapshotManager.latestSnapshotId() - 2
+                                == snapshotManager.earliestSnapshotId(),
+                Duration.ofSeconds(60_000),
+                Duration.ofSeconds(100),
+                String.format("Cannot validate snapshot expiration in %s milliseconds.", 60_000));
     }
 
     private List<Map<String, String>> getSpecifiedPartitions() {
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
index f16c90af..72890569 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.EnvironmentSettings;
 import org.apache.flink.table.api.TableEnvironment;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.table.store.CoreOptions;
 import org.apache.flink.table.store.connector.ReadWriteTableITCase;
 import org.apache.flink.table.store.connector.StreamingReadWriteTableWithKafkaLogITCase;
 import org.apache.flink.table.store.file.utils.BlockingIterator;
@@ -44,6 +45,12 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 import static org.apache.flink.table.planner.factories.TestValuesTableFactory.registerData;
+import static org.apache.flink.table.store.CoreOptions.SCAN_MODE;
+import static org.apache.flink.table.store.connector.FlinkConnectorOptions.LOG_SYSTEM;
+import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
+import static org.apache.flink.table.store.kafka.KafkaLogOptions.TOPIC;
+import static org.apache.flink.table.store.kafka.KafkaTableTestBase.createTopicIfNotExists;
+import static org.apache.flink.table.store.kafka.KafkaTableTestBase.getBootstrapServers;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
@@ -55,6 +62,13 @@ public class ReadWriteTableTestUtil {
 
     public static final int DEFAULT_PARALLELISM = 2;
 
+    public static final Map<String, String> SCAN_LATEST =
+            new HashMap<String, String>() {
+                {
+                    put(SCAN_MODE.key(), CoreOptions.StartupMode.LATEST.toString());
+                }
+            };
+
     public static TableEnvironment sEnv;
 
     public static StreamExecutionEnvironment bExeEnv;
@@ -113,6 +127,32 @@ public class ReadWriteTableTestUtil {
         return table;
     }
 
+    public static String createTableWithKafkaLog(
+            List<String> fieldsSpec,
+            List<String> primaryKeys,
+            List<String> partitionKeys,
+            boolean manuallyCreateLogTable) {
+        String topic = "topic_" + UUID.randomUUID();
+        String table =
+                createTable(
+                        fieldsSpec,
+                        primaryKeys,
+                        partitionKeys,
+                        new HashMap<String, String>() {
+                            {
+                                put(LOG_SYSTEM.key(), "kafka");
+                                put(BOOTSTRAP_SERVERS.key(), getBootstrapServers());
+                                put(TOPIC.key(), topic);
+                            }
+                        });
+
+        if (manuallyCreateLogTable) {
+            createTopicIfNotExists(topic, 1);
+        }
+
+        return table;
+    }
+
     public static String createTemporaryTable(
             List<String> fieldsSpec,
             List<String> primaryKeys,
@@ -208,6 +248,7 @@ public class ReadWriteTableTestUtil {
         }
         partitionSpec.stream()
                 .map(str -> str.replaceAll(",", "/"))
+                .map(str -> str.replaceAll("null", "__DEFAULT_PARTITION__"))
                 .forEach(
                         partition -> {
                             assertThat(Paths.get(warehouse, relativeFilePath, partition)).exists();
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/kafka/KafkaTableTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/kafka/KafkaTableTestBase.java
index aff25de2..5a0e3aa1 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/kafka/KafkaTableTestBase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/kafka/KafkaTableTestBase.java
@@ -122,7 +122,7 @@ public abstract class KafkaTableTestBase extends AbstractTestBase {
         deleteTopics();
     }
 
-    public Properties getStandardProps() {
+    public static Properties getStandardProps() {
         Properties standardProps = new Properties();
         standardProps.put("bootstrap.servers", KAFKA_CONTAINER.getBootstrapServers());
         standardProps.put("group.id", "flink-tests");
@@ -134,7 +134,7 @@ public abstract class KafkaTableTestBase extends AbstractTestBase {
         return standardProps;
     }
 
-    public String getBootstrapServers() {
+    public static String getBootstrapServers() {
         return KAFKA_CONTAINER.getBootstrapServers();
     }
 
@@ -142,7 +142,7 @@ public abstract class KafkaTableTestBase extends AbstractTestBase {
         return describeExternalTopics().containsKey(topicName);
     }
 
-    protected void createTopicIfNotExists(String topicName, int numBucket) {
+    public static void createTopicIfNotExists(String topicName, int numBucket) {
         try (final AdminClient adminClient = AdminClient.create(getStandardProps())) {
             if (!adminClient.listTopics().names().get().contains(topicName)) {
                 adminClient