You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2023/02/03 13:45:31 UTC

[flink-table-store] branch master updated: [FLINK-30867] Support streaming read changes from overwrite

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git


The following commit(s) were added to refs/heads/master by this push:
     new 0b2662dd [FLINK-30867] Support streaming read changes from overwrite
0b2662dd is described below

commit 0b2662dd9f31b6310b645e570528fed20722bec5
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Fri Feb 3 21:45:26 2023 +0800

    [FLINK-30867] Support streaming read changes from overwrite
    
    This closes #493
---
 .../shortcodes/generated/core_configuration.html   |   6 +
 .../store/connector/ReadWriteTableITCase.java      | 146 ++++++++++++++++-----
 .../StreamingReadWriteTableWithKafkaLogITCase.java |   3 +-
 .../connector/action/CompactActionITCase.java      |  77 ++++++-----
 .../source/FileStoreSourceSplitGeneratorTest.java  |   6 +-
 .../connector/util/ReadWriteTableTestUtil.java     |  29 ++--
 .../org/apache/flink/table/store/CoreOptions.java  |  18 ++-
 .../apache/flink/table/store/file/KeyValue.java    |   5 +
 .../table/store/file/operation/FileStoreScan.java  |   2 +-
 .../file/operation/KeyValueFileStoreRead.java      |   5 +-
 .../table/store/file/operation/ReverseReader.java  |  76 +++++++++++
 .../store/table/AppendOnlyFileStoreTable.java      |  13 ++
 .../table/ChangelogValueCountFileStoreTable.java   |  13 ++
 .../table/ChangelogWithKeyFileStoreTable.java      |   5 +
 .../store/table/source/AbstractDataTableScan.java  |  52 +++++++-
 .../flink/table/store/table/source/DataSplit.java  |  41 +++++-
 .../table/store/table/source/DataTableScan.java    |   5 +
 .../ContinuousDataFileSnapshotEnumerator.java      |  10 +-
 .../table/store/table/system/AuditLogTable.java    |  10 ++
 19 files changed, 415 insertions(+), 107 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index feae87cf..e8a4ffda 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -242,6 +242,12 @@
             <td>MemorySize</td>
             <td>Target size of a source split when scanning a bucket.</td>
         </tr>
+        <tr>
+            <td><h5>streaming-read-overwrite</h5></td>
+            <td style="word-wrap: break-word;">false</td>
+            <td>Boolean</td>
+            <td>Whether to read the changes from overwrite in streaming mode.</td>
+        </tr>
         <tr>
             <td><h5>target-file-size</h5></td>
             <td style="word-wrap: break-word;">128 mb</td>
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
index 25dd920f..4e9cce6e 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
@@ -20,6 +20,7 @@ package org.apache.flink.table.store.connector;
 
 import org.apache.flink.api.dag.Transformation;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.testutils.FlinkAssertions;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.DataStreamSink;
 import org.apache.flink.streaming.api.transformations.PartitionTransformation;
@@ -59,7 +60,6 @@ import static org.apache.flink.table.planner.factories.TestValuesTableFactory.ch
 import static org.apache.flink.table.store.connector.AbstractTableStoreFactory.buildFileStoreTable;
 import static org.apache.flink.table.store.connector.FlinkConnectorOptions.SCAN_PARALLELISM;
 import static org.apache.flink.table.store.connector.FlinkConnectorOptions.SINK_PARALLELISM;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestBase.assertNoMoreRecords;
 import static org.apache.flink.table.store.connector.TableStoreTestBase.createResolvedTable;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.bEnv;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.bExeEnv;
@@ -77,12 +77,20 @@ import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.sEnv;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testBatchRead;
 import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingRead;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.validateStreamingReadResult;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 /** Table Store reading and writing IT cases. */
 public class ReadWriteTableITCase extends AbstractTestBase {
 
+    private final Map<String, String> streamingReadOverwrite =
+            new HashMap<String, String>() {
+                {
+                    put(CoreOptions.STREAMING_READ_OVERWRITE.key(), "true");
+                }
+            };
+
     @BeforeEach
     public void setUp() {
         init(getTempDirPath());
@@ -120,17 +128,9 @@ public class ReadWriteTableITCase extends AbstractTestBase {
 
         testBatchRead(buildSimpleQuery(table), initialRecords);
 
-        BlockingIterator<Row, Row> streamItr =
-                testStreamingRead(buildSimpleQuery(table), initialRecords);
-
-        // test refresh after overwriting
         insertOverwritePartition(
                 table, "PARTITION (dt = '2022-01-02')", "('Euro', 100)", "('Yen', 1)");
 
-        // streaming iterator will not receive any changelog
-        assertNoMoreRecords(streamItr);
-        streamItr.close();
-
         // batch read to check partition refresh
         testBatchRead(
                 buildQuery(table, "*", "WHERE dt IN ('2022-01-02')"),
@@ -223,27 +223,6 @@ public class ReadWriteTableITCase extends AbstractTestBase {
 
         testBatchRead(buildSimpleQuery(table), initialRecords);
 
-        // overwrite
-        insertOverwrite(table, "('Euro', 90, '2022-01-01')", "('Yen', 2, '2022-01-02')");
-
-        // test streaming read
-        testStreamingRead(
-                        buildSimpleQuery(table),
-                        Arrays.asList(
-                                changelogRow("+I", "Euro", 90L, "2022-01-01"),
-                                changelogRow("+I", "Yen", 2L, "2022-01-02")))
-                .close();
-
-        // overwrite with initial data
-        insertOverwrite(
-                table,
-                "('US Dollar', 102, '2022-01-01')",
-                "('Euro', 114, '2022-01-01')",
-                "('Yen', 1, '2022-01-01')",
-                "('Euro', 114, '2022-01-01')",
-                "('US Dollar', 114, '2022-01-01')",
-                "('Euro', 119, '2022-01-02')");
-
         // test partition filter
         testBatchRead(buildQuery(table, "*", "WHERE dt >= '2022-01-01'"), initialRecords);
 
@@ -426,9 +405,6 @@ public class ReadWriteTableITCase extends AbstractTestBase {
 
         insertIntoFromTable(temporaryTable, table);
 
-        sEnv.executeSql(String.format("INSERT INTO `%s` SELECT * FROM `%s`", table, temporaryTable))
-                .await();
-
         checkFileStorePath(table, Arrays.asList("dt=2022-01-01", "dt=2022-01-02"));
 
         testStreamingRead(
@@ -741,6 +717,107 @@ public class ReadWriteTableITCase extends AbstractTestBase {
                         changelogRow("+I", 1, "2023-01-02", "store")));
     }
 
+    // ----------------------------------------------------------------------------------------------------------------
+    // Streaming Read of Overwrite
+    // ----------------------------------------------------------------------------------------------------------------
+
+    @Test
+    public void testStreamingReadOverwriteWithPartitionedRecords() throws Exception {
+        String table =
+                createTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt String"),
+                        Arrays.asList("currency", "dt"),
+                        Collections.singletonList("dt"),
+                        streamingReadOverwrite);
+
+        insertInto(
+                table,
+                "('US Dollar', 114, '2022-01-01')",
+                "('Yen', 1, '2022-01-01')",
+                "('Euro', 114, '2022-01-01')",
+                "('Euro', 119, '2022-01-02')");
+
+        checkFileStorePath(table, Arrays.asList("dt=2022-01-01", "dt=2022-01-02"));
+
+        // test reading after overwriting
+        insertOverwritePartition(table, "PARTITION (dt = '2022-01-01')", "('US Dollar', 120)");
+
+        BlockingIterator<Row, Row> streamingItr =
+                testStreamingRead(
+                        buildSimpleQuery(table),
+                        Arrays.asList(
+                                // part = 2022-01-01
+                                changelogRow("+I", "US Dollar", 120L, "2022-01-01"),
+                                // part = 2022-01-02
+                                changelogRow("+I", "Euro", 119L, "2022-01-02")));
+
+        // test refresh after overwriting
+        insertOverwritePartition(
+                table, "PARTITION (dt = '2022-01-02')", "('Euro', 100)", "('Yen', 1)");
+
+        validateStreamingReadResult(
+                streamingItr,
+                Arrays.asList(
+                        // part = 2022-01-02
+                        changelogRow("-D", "Euro", 119L, "2022-01-02"),
+                        changelogRow("+I", "Euro", 100L, "2022-01-02"),
+                        changelogRow("+I", "Yen", 1L, "2022-01-02")));
+
+        streamingItr.close();
+    }
+
+    @Test
+    public void testStreamingReadOverwriteWithoutPartitionedRecords() throws Exception {
+        String table =
+                createTable(
+                        Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
+                        Collections.singletonList("currency"),
+                        Collections.emptyList(),
+                        streamingReadOverwrite);
+
+        insertInto(
+                table,
+                "('US Dollar', 102, '2022-01-01')",
+                "('Yen', 1, '2022-01-02')",
+                "('Euro', 119, '2022-01-02')");
+
+        checkFileStorePath(table, Collections.emptyList());
+
+        // test projection and filter
+        BlockingIterator<Row, Row> streamingItr =
+                testStreamingRead(
+                        buildQuery(table, "currency, rate", "WHERE dt = '2022-01-02'"),
+                        Arrays.asList(
+                                changelogRow("+I", "Yen", 1L), changelogRow("+I", "Euro", 119L)));
+
+        insertOverwrite(table, "('US Dollar', 100, '2022-01-02')", "('Yen', 10, '2022-01-01')");
+
+        validateStreamingReadResult(
+                streamingItr,
+                Arrays.asList(
+                        changelogRow("-D", "Yen", 1L),
+                        changelogRow("-D", "Euro", 119L),
+                        changelogRow("+I", "US Dollar", 100L)));
+
+        streamingItr.close();
+    }
+
+    @Test
+    public void testUnsupportStreamingReadOverwriteWithoutPk() {
+        assertThatThrownBy(
+                        () ->
+                                createTable(
+                                        Arrays.asList(
+                                                "currency STRING", "rate BIGINT", "dt String"),
+                                        Collections.emptyList(),
+                                        Collections.singletonList("dt"),
+                                        streamingReadOverwrite))
+                .satisfies(
+                        FlinkAssertions.anyCauseMatches(
+                                RuntimeException.class,
+                                "Doesn't support streaming read the changes from overwrite when the primary keys are not defined."));
+    }
+
     // ----------------------------------------------------------------------------------------------------------------
     // Keyword
     // ----------------------------------------------------------------------------------------------------------------
@@ -782,7 +859,8 @@ public class ReadWriteTableITCase extends AbstractTestBase {
                         changelogRow("+I", 2, "test%2"),
                         changelogRow("+I", 8, "tests"),
                         changelogRow("+I", 10, "tested"),
-                        changelogRow("+I", 20, "test_123")));
+                        changelogRow("+I", 20, "test_123"),
+                        changelogRow("+I", 100, "test%fff")));
 
         testBatchRead(
                 buildQuery(table, "*", "WHERE f1 LIKE 'v%'"),
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
index 7061f087..677c15eb 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
@@ -144,6 +144,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
 
         // check no changelog generated for streaming read
         assertNoMoreRecords(streamItr);
+        streamItr.close();
 
         // batch read to check data refresh
         testBatchRead(
@@ -156,8 +157,6 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
                         changelogRow("+I", "Yen", 20L, "2022-01-03"),
                         changelogRow("+I", "Yen", 20L, "2022-01-04")));
 
-        streamItr.close();
-
         // filter on partition
         testStreamingRead(
                         buildQuery(table, "*", "WHERE dt = '2022-01-01'"),
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
index 4dd94b51..22279f28 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
@@ -42,6 +42,8 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeoutException;
 
 /** IT cases for {@link CompactAction}. */
 public class CompactActionITCase extends ActionITCaseBase {
@@ -53,7 +55,7 @@ public class CompactActionITCase extends ActionITCaseBase {
             RowType.of(FIELD_TYPES, new String[] {"k", "v", "hh", "dt"});
 
     @Test
-    @Timeout(60000)
+    @Timeout(60)
     public void testBatchCompact() throws Exception {
         Map<String, String> options = new HashMap<>();
         options.put(CoreOptions.WRITE_ONLY.key(), "true");
@@ -84,7 +86,7 @@ public class CompactActionITCase extends ActionITCaseBase {
 
         StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
         env.setRuntimeMode(RuntimeExecutionMode.BATCH);
-        env.setParallelism(2);
+        env.setParallelism(ThreadLocalRandom.current().nextInt(2) + 1);
         new CompactAction(tablePath).withPartitions(getSpecifiedPartitions()).build(env);
         env.execute();
 
@@ -106,7 +108,6 @@ public class CompactActionITCase extends ActionITCaseBase {
     }
 
     @Test
-    @Timeout(60000)
     public void testStreamingCompact() throws Exception {
         Map<String, String> options = new HashMap<>();
         options.put(CoreOptions.CHANGELOG_PRODUCER.key(), "full-compaction");
@@ -148,27 +149,16 @@ public class CompactActionITCase extends ActionITCaseBase {
         env.setRuntimeMode(RuntimeExecutionMode.STREAMING);
         env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
         env.getCheckpointConfig().setCheckpointInterval(500);
-        env.setParallelism(2);
+        env.setParallelism(ThreadLocalRandom.current().nextInt(2) + 1);
         new CompactAction(tablePath).withPartitions(getSpecifiedPartitions()).build(env);
         env.executeAsync();
 
-        while (true) {
-            plan = snapshotEnumerator.enumerate();
-            if (plan != null) {
-                break;
-            }
-            Thread.sleep(1000);
-        }
-
         // first full compaction
-        List<String> actual = new ArrayList<>();
-        while (plan != null) {
-            actual.addAll(getResult(table.newRead(), plan.splits(), ROW_TYPE));
-            plan = snapshotEnumerator.enumerate();
-        }
-        actual.sort(String::compareTo);
-        Assertions.assertEquals(
-                Arrays.asList("+I[1, 100, 15, 20221208]", "+I[1, 100, 15, 20221209]"), actual);
+        validateResult(
+                table,
+                snapshotEnumerator,
+                Arrays.asList("+I[1, 100, 15, 20221208]", "+I[1, 100, 15, 20221209]"),
+                60_000);
 
         // incremental records
         writeData(
@@ -176,28 +166,16 @@ public class CompactActionITCase extends ActionITCaseBase {
                 rowData(1, 101, 16, BinaryString.fromString("20221208")),
                 rowData(1, 101, 15, BinaryString.fromString("20221209")));
 
-        while (true) {
-            plan = snapshotEnumerator.enumerate();
-            if (plan != null) {
-                break;
-            }
-            Thread.sleep(1000);
-        }
-
         // second full compaction
-        actual = new ArrayList<>();
-        while (plan != null) {
-            actual.addAll(getResult(table.newRead(), plan.splits(), ROW_TYPE));
-            plan = snapshotEnumerator.enumerate();
-        }
-        actual.sort(String::compareTo);
-        Assertions.assertEquals(
+        validateResult(
+                table,
+                snapshotEnumerator,
                 Arrays.asList(
                         "+U[1, 101, 15, 20221208]",
                         "+U[1, 101, 15, 20221209]",
                         "-U[1, 100, 15, 20221208]",
                         "-U[1, 100, 15, 20221209]"),
-                actual);
+                60_000);
 
         // assert dedicated compact job will expire snapshots
         Assertions.assertEquals(
@@ -216,4 +194,31 @@ public class CompactActionITCase extends ActionITCaseBase {
 
         return Arrays.asList(partition1, partition2);
     }
+
+    private void validateResult(
+            FileStoreTable table,
+            SnapshotEnumerator snapshotEnumerator,
+            List<String> expected,
+            long timeout)
+            throws Exception {
+        List<String> actual = new ArrayList<>();
+        long start = System.currentTimeMillis();
+        while (actual.size() != expected.size()) {
+            DataTableScan.DataFilePlan plan = snapshotEnumerator.enumerate();
+            if (plan != null) {
+                actual.addAll(getResult(table.newRead(), plan.splits(), ROW_TYPE));
+            }
+            if (System.currentTimeMillis() - start > timeout) {
+                break;
+            }
+        }
+        if (actual.size() != expected.size()) {
+            throw new TimeoutException(
+                    String.format(
+                            "Cannot collect %s records in %s milliseconds.",
+                            expected.size(), timeout));
+        }
+        actual.sort(String::compareTo);
+        Assertions.assertEquals(expected, actual);
+    }
 }
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
index 8d7b1937..be106268 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
@@ -28,7 +28,6 @@ import org.apache.flink.table.store.table.source.DataSplit;
 import org.apache.flink.table.store.table.source.DataTableScan;
 
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
 
 import javax.annotation.Nullable;
 
@@ -44,8 +43,6 @@ import static org.assertj.core.api.Assertions.assertThat;
 /** Test for {@link FileStoreSourceSplitGenerator}. */
 public class FileStoreSourceSplitGeneratorTest {
 
-    @TempDir java.nio.file.Path tempDir;
-
     @Test
     public void test() {
         FileStoreScan.Plan plan =
@@ -80,8 +77,9 @@ public class FileStoreSourceSplitGeneratorTest {
                 AbstractDataTableScan.generateSplits(
                         1L,
                         false,
+                        false,
                         Collections::singletonList,
-                        plan.groupByPartFiles(plan.files(FileKind.ADD)));
+                        FileStoreScan.Plan.groupByPartFiles(plan.files(FileKind.ADD)));
         DataTableScan.DataFilePlan tableScanPlan = new DataTableScan.DataFilePlan(1L, scanSplits);
 
         List<FileStoreSourceSplit> splits =
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
index 709f6594..f16c90af 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
@@ -225,54 +225,47 @@ public class ReadWriteTableTestUtil {
     public static void testBatchRead(String query, List<Row> expected) throws Exception {
         CloseableIterator<Row> resultItr = bEnv.executeSql(query).collect();
         try (BlockingIterator<Row, Row> iterator = BlockingIterator.of(resultItr)) {
-            if (expected.isEmpty()) {
-                assertThat(resultItr.hasNext()).isFalse();
-            } else {
+            if (!expected.isEmpty()) {
                 assertThat(
                                 iterator.collect(
                                         expected.size(), TIME_OUT.getSize(), TIME_OUT.getUnit()))
                         .containsExactlyInAnyOrderElementsOf(expected);
             }
+            assertThat(resultItr.hasNext()).isFalse();
         }
     }
 
     public static BlockingIterator<Row, Row> testStreamingRead(String query, List<Row> expected)
             throws Exception {
         BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
-
-        if (expected.isEmpty()) {
-            assertNoMoreRecords(iterator);
-        } else {
-            assertThat(iterator.collect(expected.size()))
-                    .containsExactlyInAnyOrderElementsOf(expected);
-        }
-
+        validateStreamingReadResult(iterator, expected);
         return iterator;
     }
 
     public static BlockingIterator<Row, Row> testStreamingReadWithReadFirst(
             String source, String sink, String query, List<Row> expected) throws Exception {
         BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
-
         insertIntoFromTable(source, sink);
+        validateStreamingReadResult(iterator, expected);
+        return iterator;
+    }
 
+    public static void validateStreamingReadResult(
+            BlockingIterator<Row, Row> streamingItr, List<Row> expected) throws Exception {
         if (expected.isEmpty()) {
-            assertNoMoreRecords(iterator);
+            assertNoMoreRecords(streamingItr);
         } else {
-            assertThat(iterator.collect(expected.size()))
+            assertThat(streamingItr.collect(expected.size()))
                     .containsExactlyInAnyOrderElementsOf(expected);
         }
-
-        return iterator;
     }
 
-    public static void assertNoMoreRecords(BlockingIterator<Row, Row> iterator) throws Exception {
+    public static void assertNoMoreRecords(BlockingIterator<Row, Row> iterator) {
         List<Row> expectedRecords = Collections.emptyList();
         try {
             // set expectation size to 1 to let time pass by until timeout
             // just wait 5s to avoid too long time
             expectedRecords = iterator.collect(1, 5L, TimeUnit.SECONDS);
-            iterator.close();
         } catch (TimeoutException ignored) {
             // don't throw exception
         }
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
index 84e019a6..f31e60c1 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
@@ -401,6 +401,13 @@ public class CoreOptions implements Serializable {
                     .withDescription(
                             "Whether to create underlying storage when reading and writing the table.");
 
+    public static final ConfigOption<Boolean> STREAMING_READ_OVERWRITE =
+            ConfigOptions.key("streaming-read-overwrite")
+                    .booleanType()
+                    .defaultValue(false)
+                    .withDescription(
+                            "Whether to read the changes from overwrite in streaming mode.");
+
     private final Configuration options;
 
     public CoreOptions(Map<String, String> options) {
@@ -604,6 +611,10 @@ public class CoreOptions implements Serializable {
         return options.get(WRITE_ONLY);
     }
 
+    public boolean streamingReadOverwrite() {
+        return options.get(STREAMING_READ_OVERWRITE);
+    }
+
     /** Specifies the merge engine for table with primary key. */
     public enum MergeEngine implements DescribedEnum {
         DEDUPLICATE("deduplicate", "De-duplicate and keep the last row."),
@@ -841,7 +852,7 @@ public class CoreOptions implements Serializable {
                                     !SYSTEM_FIELD_NAMES.contains(f),
                                     String.format(
                                             "Field name[%s] in schema cannot be exist in [%s]",
-                                            f, SYSTEM_FIELD_NAMES.toString()));
+                                            f, SYSTEM_FIELD_NAMES));
                             checkState(
                                     !f.startsWith(KEY_FIELD_PREFIX),
                                     String.format(
@@ -855,6 +866,11 @@ public class CoreOptions implements Serializable {
                     "Cannot define any primary key in an append-only table. Set 'write-mode'='change-log' if "
                             + "still want to keep the primary key definition.");
         }
+
+        if (schema.primaryKeys().isEmpty() && options.streamingReadOverwrite()) {
+            throw new RuntimeException(
+                    "Doesn't support streaming read the changes from overwrite when the primary keys are not defined.");
+        }
     }
 
     private static void checkOptionExistInMode(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/KeyValue.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/KeyValue.java
index 556547dc..e979bd9e 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/KeyValue.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/KeyValue.java
@@ -73,6 +73,11 @@ public class KeyValue {
         return this;
     }
 
+    public KeyValue replaceValueKind(RowKind valueKind) {
+        this.valueKind = valueKind;
+        return this;
+    }
+
     public InternalRow key() {
         return key;
     }
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
index 5ed517a0..77fc5f31 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
@@ -72,7 +72,7 @@ public interface FileStoreScan {
         }
 
         /** Return a map group by partition and bucket. */
-        default Map<BinaryRow, Map<Integer, List<DataFileMeta>>> groupByPartFiles(
+        static Map<BinaryRow, Map<Integer, List<DataFileMeta>>> groupByPartFiles(
                 List<ManifestEntry> files) {
             Map<BinaryRow, Map<Integer, List<DataFileMeta>>> groupBy = new HashMap<>();
             for (ManifestEntry entry : files) {
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/KeyValueFileStoreRead.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/KeyValueFileStoreRead.java
index 419f6b45..af4adcba 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/KeyValueFileStoreRead.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/KeyValueFileStoreRead.java
@@ -169,7 +169,10 @@ public class KeyValueFileStoreRead implements FileStoreRead<KeyValue> {
                                     file.schemaId(), fileName, file.level());
                         });
             }
-            return ConcatRecordReader.create(suppliers);
+            RecordReader<KeyValue> concatRecordReader = ConcatRecordReader.create(suppliers);
+            return split.reverseRowKind()
+                    ? new ReverseReader(concatRecordReader)
+                    : concatRecordReader;
         } else {
             // Sections are read by SortMergeReader, which sorts and merges records by keys.
             // So we cannot project keys or else the sorting will be incorrect.
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/ReverseReader.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/ReverseReader.java
new file mode 100644
index 00000000..752aed4e
--- /dev/null
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/ReverseReader.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.operation;
+
+import org.apache.flink.table.store.file.KeyValue;
+import org.apache.flink.table.store.file.utils.RecordReader;
+import org.apache.flink.table.store.types.RowKind;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+
+/**
+ * A {@link RecordReader} which reverses the {@link RowKind} of the {@link KeyValue} from {@link
+ * RowKind#INSERT} to {@link RowKind#DELETE} read from the wrapped reader.
+ */
+public class ReverseReader implements RecordReader<KeyValue> {
+
+    private final RecordReader<KeyValue> reader;
+
+    public ReverseReader(RecordReader<KeyValue> reader) {
+        this.reader = reader;
+    }
+
+    @Nullable
+    @Override
+    public RecordIterator<KeyValue> readBatch() throws IOException {
+        RecordIterator<KeyValue> batch = reader.readBatch();
+
+        if (batch == null) {
+            return null;
+        }
+
+        return new RecordIterator<KeyValue>() {
+            @Override
+            public KeyValue next() throws IOException {
+                KeyValue kv = batch.next();
+                if (kv == null) {
+                    return null;
+                }
+                if (kv.valueKind() == RowKind.UPDATE_BEFORE || kv.valueKind() == RowKind.DELETE) {
+                    throw new IllegalStateException(
+                            "In reverse reader, the value kind of records cannot be UPDATE_BEFORE or DELETE.");
+                }
+
+                return kv.replaceValueKind(RowKind.DELETE);
+            }
+
+            @Override
+            public void releaseBatch() {
+                batch.releaseBatch();
+            }
+        };
+    }
+
+    @Override
+    public void close() throws IOException {
+        reader.close();
+    }
+}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTable.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTable.java
index f97b7550..2540f2e9 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTable.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTable.java
@@ -24,6 +24,7 @@ import org.apache.flink.table.store.file.AppendOnlyFileStore;
 import org.apache.flink.table.store.file.WriteMode;
 import org.apache.flink.table.store.file.operation.AppendOnlyFileStoreRead;
 import org.apache.flink.table.store.file.operation.AppendOnlyFileStoreScan;
+import org.apache.flink.table.store.file.operation.ReverseReader;
 import org.apache.flink.table.store.file.predicate.Predicate;
 import org.apache.flink.table.store.file.schema.TableSchema;
 import org.apache.flink.table.store.file.utils.FileStorePathFactory;
@@ -81,6 +82,18 @@ public class AppendOnlyFileStoreTable extends AbstractFileStoreTable {
         AppendOnlyFileStoreScan scan = store().newScan();
         return new AbstractDataTableScan(
                 fileIO, scan, tableSchema, store().pathFactory(), options()) {
+            /**
+             * Currently, the streaming read of overwrite is implemented by reversing the {@link
+             * RowKind} of overwrote records to {@link RowKind#DELETE}, so only tables that have
+             * primary key support it.
+             *
+             * @see ReverseReader
+             */
+            @Override
+            public boolean supportStreamingReadOverwrite() {
+                return false;
+            }
+
             @Override
             protected SplitGenerator splitGenerator(FileStorePathFactory pathFactory) {
                 return new AppendOnlySplitGenerator(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogValueCountFileStoreTable.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogValueCountFileStoreTable.java
index f20d9cbe..8181e249 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogValueCountFileStoreTable.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogValueCountFileStoreTable.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.store.file.WriteMode;
 import org.apache.flink.table.store.file.io.DataFileMeta;
 import org.apache.flink.table.store.file.mergetree.compact.ValueCountMergeFunction;
 import org.apache.flink.table.store.file.operation.KeyValueFileStoreScan;
+import org.apache.flink.table.store.file.operation.ReverseReader;
 import org.apache.flink.table.store.file.predicate.Predicate;
 import org.apache.flink.table.store.file.schema.KeyValueFieldsExtractor;
 import org.apache.flink.table.store.file.schema.TableSchema;
@@ -96,6 +97,18 @@ public class ChangelogValueCountFileStoreTable extends AbstractFileStoreTable {
         KeyValueFileStoreScan scan = store().newScan();
         return new AbstractDataTableScan(
                 fileIO, scan, tableSchema, store().pathFactory(), options()) {
+            /**
+             * Currently, the streaming read of overwrite is implemented by reversing the {@link
+             * RowKind} of overwrote records to {@link RowKind#DELETE}, so only tables that have
+             * primary key support it.
+             *
+             * @see ReverseReader
+             */
+            @Override
+            public boolean supportStreamingReadOverwrite() {
+                return false;
+            }
+
             @Override
             protected SplitGenerator splitGenerator(FileStorePathFactory pathFactory) {
                 return new MergeTreeSplitGenerator(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogWithKeyFileStoreTable.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogWithKeyFileStoreTable.java
index b7817133..00a68d60 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogWithKeyFileStoreTable.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogWithKeyFileStoreTable.java
@@ -152,6 +152,11 @@ public class ChangelogWithKeyFileStoreTable extends AbstractFileStoreTable {
         KeyValueFileStoreScan scan = store().newScan();
         return new AbstractDataTableScan(
                 fileIO(), scan, tableSchema, store().pathFactory(), options()) {
+            @Override
+            public boolean supportStreamingReadOverwrite() {
+                return new CoreOptions(tableSchema.options()).streamingReadOverwrite();
+            }
+
             @Override
             protected SplitGenerator splitGenerator(FileStorePathFactory pathFactory) {
                 return new MergeTreeSplitGenerator(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
index 632787c0..4c020518 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
@@ -126,8 +126,43 @@ public abstract class AbstractDataTableScan implements DataTableScan {
                 generateSplits(
                         snapshotId == null ? Snapshot.FIRST_SNAPSHOT_ID - 1 : snapshotId,
                         scanKind != ScanKind.ALL,
+                        false,
                         splitGenerator(pathFactory),
-                        plan.groupByPartFiles(plan.files(FileKind.ADD)));
+                        FileStoreScan.Plan.groupByPartFiles(plan.files(FileKind.ADD)));
+        return new DataFilePlan(snapshotId, splits);
+    }
+
+    @Override
+    public DataFilePlan planOverwriteChanges() {
+        withKind(ScanKind.DELTA);
+        FileStoreScan.Plan plan = scan.plan();
+        long snapshotId = plan.snapshotId();
+
+        SnapshotManager snapshotManager = new SnapshotManager(fileIO, pathFactory.root());
+        Snapshot snapshot = snapshotManager.snapshot(snapshotId);
+        if (snapshot.commitKind() != Snapshot.CommitKind.OVERWRITE) {
+            throw new IllegalStateException(
+                    "Cannot plan overwrite changes from a non-overwrite snapshot.");
+        }
+
+        List<DataSplit> splits = new ArrayList<>();
+
+        splits.addAll(
+                generateSplits(
+                        snapshotId,
+                        true,
+                        true,
+                        splitGenerator(pathFactory),
+                        FileStoreScan.Plan.groupByPartFiles(plan.files(FileKind.DELETE))));
+
+        splits.addAll(
+                generateSplits(
+                        snapshotId,
+                        true,
+                        false,
+                        splitGenerator(pathFactory),
+                        FileStoreScan.Plan.groupByPartFiles(plan.files(FileKind.ADD))));
+
         return new DataFilePlan(snapshotId, splits);
     }
 
@@ -135,6 +170,7 @@ public abstract class AbstractDataTableScan implements DataTableScan {
     public static List<DataSplit> generateSplits(
             long snapshotId,
             boolean isIncremental,
+            boolean reverseRowKind,
             SplitGenerator splitGenerator,
             Map<BinaryRow, Map<Integer, List<DataFileMeta>>> groupedDataFiles) {
         List<DataSplit> splits = new ArrayList<>();
@@ -148,13 +184,23 @@ public abstract class AbstractDataTableScan implements DataTableScan {
                     // Don't split when incremental
                     splits.add(
                             new DataSplit(
-                                    snapshotId, partition, bucket, bucketEntry.getValue(), true));
+                                    snapshotId,
+                                    partition,
+                                    bucket,
+                                    bucketEntry.getValue(),
+                                    true,
+                                    reverseRowKind));
                 } else {
                     splitGenerator.split(bucketEntry.getValue()).stream()
                             .map(
                                     files ->
                                             new DataSplit(
-                                                    snapshotId, partition, bucket, files, false))
+                                                    snapshotId,
+                                                    partition,
+                                                    bucket,
+                                                    files,
+                                                    false,
+                                                    reverseRowKind))
                             .forEach(splits::add);
                 }
             }
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataSplit.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataSplit.java
index 37e5c592..7e354c66 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataSplit.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataSplit.java
@@ -45,13 +45,27 @@ public class DataSplit implements Split {
     private List<DataFileMeta> files;
     private boolean isIncremental;
 
+    // when reverseRowKind is true, the RowKind of records from this split should be reversed to
+    // DELETE
+    private boolean reverseRowKind;
+
     public DataSplit(
             long snapshotId,
             BinaryRow partition,
             int bucket,
             List<DataFileMeta> files,
             boolean isIncremental) {
-        init(snapshotId, partition, bucket, files, isIncremental);
+        init(snapshotId, partition, bucket, files, isIncremental, false);
+    }
+
+    public DataSplit(
+            long snapshotId,
+            BinaryRow partition,
+            int bucket,
+            List<DataFileMeta> files,
+            boolean isIncremental,
+            boolean reverseRowKind) {
+        init(snapshotId, partition, bucket, files, isIncremental, reverseRowKind);
     }
 
     private void init(
@@ -59,12 +73,14 @@ public class DataSplit implements Split {
             BinaryRow partition,
             int bucket,
             List<DataFileMeta> files,
-            boolean isIncremental) {
+            boolean isIncremental,
+            boolean reverseRowKind) {
         this.snapshotId = snapshotId;
         this.partition = partition;
         this.bucket = bucket;
         this.files = files;
         this.isIncremental = isIncremental;
+        this.reverseRowKind = reverseRowKind;
     }
 
     public long snapshotId() {
@@ -87,6 +103,10 @@ public class DataSplit implements Split {
         return isIncremental;
     }
 
+    public boolean reverseRowKind() {
+        return reverseRowKind;
+    }
+
     @Override
     public long rowCount() {
         long rowCount = 0;
@@ -108,12 +128,13 @@ public class DataSplit implements Split {
         return bucket == split.bucket
                 && Objects.equals(partition, split.partition)
                 && Objects.equals(files, split.files)
-                && isIncremental == split.isIncremental;
+                && isIncremental == split.isIncremental
+                && reverseRowKind == split.reverseRowKind;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(partition, bucket, files, isIncremental);
+        return Objects.hash(partition, bucket, files, isIncremental, reverseRowKind);
     }
 
     private void writeObject(ObjectOutputStream out) throws IOException {
@@ -122,7 +143,13 @@ public class DataSplit implements Split {
 
     private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
         DataSplit split = DataSplit.deserialize(new DataInputViewStreamWrapper(in));
-        init(split.snapshotId, split.partition, split.bucket, split.files, split.isIncremental);
+        init(
+                split.snapshotId,
+                split.partition,
+                split.bucket,
+                split.files,
+                split.isIncremental,
+                split.reverseRowKind);
     }
 
     public void serialize(DataOutputView out) throws IOException {
@@ -135,6 +162,7 @@ public class DataSplit implements Split {
             dataFileSer.serialize(file, out);
         }
         out.writeBoolean(isIncremental);
+        out.writeBoolean(reverseRowKind);
     }
 
     public static DataSplit deserialize(DataInputView in) throws IOException {
@@ -147,6 +175,7 @@ public class DataSplit implements Split {
         for (int i = 0; i < fileNumber; i++) {
             files.add(dataFileSer.deserialize(in));
         }
-        return new DataSplit(snapshotId, partition, bucket, files, in.readBoolean());
+        return new DataSplit(
+                snapshotId, partition, bucket, files, in.readBoolean(), in.readBoolean());
     }
 }
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataTableScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataTableScan.java
index 5509929f..cfcc0f9f 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataTableScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataTableScan.java
@@ -40,6 +40,11 @@ public interface DataTableScan extends TableScan {
 
     DataTableScan.DataFilePlan plan();
 
+    /** to extract plan from overwrite changes. */
+    DataTableScan.DataFilePlan planOverwriteChanges();
+
+    boolean supportStreamingReadOverwrite();
+
     /** Scanning plan containing snapshot ID and input splits. */
     class DataFilePlan implements Plan {
 
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/snapshot/ContinuousDataFileSnapshotEnumerator.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/snapshot/ContinuousDataFileSnapshotEnumerator.java
index 8fbf244d..41d3811a 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/snapshot/ContinuousDataFileSnapshotEnumerator.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/snapshot/ContinuousDataFileSnapshotEnumerator.java
@@ -95,7 +95,15 @@ public class ContinuousDataFileSnapshotEnumerator implements SnapshotEnumerator
 
             Snapshot snapshot = snapshotManager.snapshot(nextSnapshotId);
 
-            if (followUpScanner.shouldScanSnapshot(snapshot)) {
+            // first check changes of overwrite
+            if (snapshot.commitKind() == Snapshot.CommitKind.OVERWRITE
+                    && scan.supportStreamingReadOverwrite()) {
+                LOG.debug("Find overwrite snapshot id {}.", nextSnapshotId);
+                DataTableScan.DataFilePlan overwritePlan =
+                        scan.withSnapshot(nextSnapshotId).planOverwriteChanges();
+                nextSnapshotId++;
+                return overwritePlan;
+            } else if (followUpScanner.shouldScanSnapshot(snapshot)) {
                 LOG.debug("Find snapshot id {}.", nextSnapshotId);
                 DataTableScan.DataFilePlan plan = followUpScanner.getPlan(nextSnapshotId, scan);
                 nextSnapshotId++;
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/system/AuditLogTable.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/system/AuditLogTable.java
index f75f6ad0..6849269d 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/system/AuditLogTable.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/system/AuditLogTable.java
@@ -180,6 +180,16 @@ public class AuditLogTable implements DataTable {
         public DataTableScan.DataFilePlan plan() {
             return dataScan.plan();
         }
+
+        @Override
+        public DataFilePlan planOverwriteChanges() {
+            return dataScan.planOverwriteChanges();
+        }
+
+        @Override
+        public boolean supportStreamingReadOverwrite() {
+            return dataScan.supportStreamingReadOverwrite();
+        }
     }
 
     private class AuditLogRead implements TableRead {