You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2023/02/03 13:45:31 UTC
[flink-table-store] branch master updated: [FLINK-30867] Support streaming read changes from overwrite
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
The following commit(s) were added to refs/heads/master by this push:
new 0b2662dd [FLINK-30867] Support streaming read changes from overwrite
0b2662dd is described below
commit 0b2662dd9f31b6310b645e570528fed20722bec5
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Fri Feb 3 21:45:26 2023 +0800
[FLINK-30867] Support streaming read changes from overwrite
This closes #493
---
.../shortcodes/generated/core_configuration.html | 6 +
.../store/connector/ReadWriteTableITCase.java | 146 ++++++++++++++++-----
.../StreamingReadWriteTableWithKafkaLogITCase.java | 3 +-
.../connector/action/CompactActionITCase.java | 77 ++++++-----
.../source/FileStoreSourceSplitGeneratorTest.java | 6 +-
.../connector/util/ReadWriteTableTestUtil.java | 29 ++--
.../org/apache/flink/table/store/CoreOptions.java | 18 ++-
.../apache/flink/table/store/file/KeyValue.java | 5 +
.../table/store/file/operation/FileStoreScan.java | 2 +-
.../file/operation/KeyValueFileStoreRead.java | 5 +-
.../table/store/file/operation/ReverseReader.java | 76 +++++++++++
.../store/table/AppendOnlyFileStoreTable.java | 13 ++
.../table/ChangelogValueCountFileStoreTable.java | 13 ++
.../table/ChangelogWithKeyFileStoreTable.java | 5 +
.../store/table/source/AbstractDataTableScan.java | 52 +++++++-
.../flink/table/store/table/source/DataSplit.java | 41 +++++-
.../table/store/table/source/DataTableScan.java | 5 +
.../ContinuousDataFileSnapshotEnumerator.java | 10 +-
.../table/store/table/system/AuditLogTable.java | 10 ++
19 files changed, 415 insertions(+), 107 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index feae87cf..e8a4ffda 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -242,6 +242,12 @@
<td>MemorySize</td>
<td>Target size of a source split when scanning a bucket.</td>
</tr>
+ <tr>
+ <td><h5>streaming-read-overwrite</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to read the changes from overwrite in streaming mode.</td>
+ </tr>
<tr>
<td><h5>target-file-size</h5></td>
<td style="word-wrap: break-word;">128 mb</td>
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
index 25dd920f..4e9cce6e 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
@@ -20,6 +20,7 @@ package org.apache.flink.table.store.connector;
import org.apache.flink.api.dag.Transformation;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.testutils.FlinkAssertions;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSink;
import org.apache.flink.streaming.api.transformations.PartitionTransformation;
@@ -59,7 +60,6 @@ import static org.apache.flink.table.planner.factories.TestValuesTableFactory.ch
import static org.apache.flink.table.store.connector.AbstractTableStoreFactory.buildFileStoreTable;
import static org.apache.flink.table.store.connector.FlinkConnectorOptions.SCAN_PARALLELISM;
import static org.apache.flink.table.store.connector.FlinkConnectorOptions.SINK_PARALLELISM;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestBase.assertNoMoreRecords;
import static org.apache.flink.table.store.connector.TableStoreTestBase.createResolvedTable;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.bEnv;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.bExeEnv;
@@ -77,12 +77,20 @@ import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.sEnv;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testBatchRead;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingRead;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.validateStreamingReadResult;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Table Store reading and writing IT cases. */
public class ReadWriteTableITCase extends AbstractTestBase {
+ private final Map<String, String> streamingReadOverwrite =
+ new HashMap<String, String>() {
+ {
+ put(CoreOptions.STREAMING_READ_OVERWRITE.key(), "true");
+ }
+ };
+
@BeforeEach
public void setUp() {
init(getTempDirPath());
@@ -120,17 +128,9 @@ public class ReadWriteTableITCase extends AbstractTestBase {
testBatchRead(buildSimpleQuery(table), initialRecords);
- BlockingIterator<Row, Row> streamItr =
- testStreamingRead(buildSimpleQuery(table), initialRecords);
-
- // test refresh after overwriting
insertOverwritePartition(
table, "PARTITION (dt = '2022-01-02')", "('Euro', 100)", "('Yen', 1)");
- // streaming iterator will not receive any changelog
- assertNoMoreRecords(streamItr);
- streamItr.close();
-
// batch read to check partition refresh
testBatchRead(
buildQuery(table, "*", "WHERE dt IN ('2022-01-02')"),
@@ -223,27 +223,6 @@ public class ReadWriteTableITCase extends AbstractTestBase {
testBatchRead(buildSimpleQuery(table), initialRecords);
- // overwrite
- insertOverwrite(table, "('Euro', 90, '2022-01-01')", "('Yen', 2, '2022-01-02')");
-
- // test streaming read
- testStreamingRead(
- buildSimpleQuery(table),
- Arrays.asList(
- changelogRow("+I", "Euro", 90L, "2022-01-01"),
- changelogRow("+I", "Yen", 2L, "2022-01-02")))
- .close();
-
- // overwrite with initial data
- insertOverwrite(
- table,
- "('US Dollar', 102, '2022-01-01')",
- "('Euro', 114, '2022-01-01')",
- "('Yen', 1, '2022-01-01')",
- "('Euro', 114, '2022-01-01')",
- "('US Dollar', 114, '2022-01-01')",
- "('Euro', 119, '2022-01-02')");
-
// test partition filter
testBatchRead(buildQuery(table, "*", "WHERE dt >= '2022-01-01'"), initialRecords);
@@ -426,9 +405,6 @@ public class ReadWriteTableITCase extends AbstractTestBase {
insertIntoFromTable(temporaryTable, table);
- sEnv.executeSql(String.format("INSERT INTO `%s` SELECT * FROM `%s`", table, temporaryTable))
- .await();
-
checkFileStorePath(table, Arrays.asList("dt=2022-01-01", "dt=2022-01-02"));
testStreamingRead(
@@ -741,6 +717,107 @@ public class ReadWriteTableITCase extends AbstractTestBase {
changelogRow("+I", 1, "2023-01-02", "store")));
}
+ // ----------------------------------------------------------------------------------------------------------------
+ // Streaming Read of Overwrite
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Test
+ public void testStreamingReadOverwriteWithPartitionedRecords() throws Exception {
+ String table =
+ createTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt String"),
+ Arrays.asList("currency", "dt"),
+ Collections.singletonList("dt"),
+ streamingReadOverwrite);
+
+ insertInto(
+ table,
+ "('US Dollar', 114, '2022-01-01')",
+ "('Yen', 1, '2022-01-01')",
+ "('Euro', 114, '2022-01-01')",
+ "('Euro', 119, '2022-01-02')");
+
+ checkFileStorePath(table, Arrays.asList("dt=2022-01-01", "dt=2022-01-02"));
+
+ // test reading after overwriting
+ insertOverwritePartition(table, "PARTITION (dt = '2022-01-01')", "('US Dollar', 120)");
+
+ BlockingIterator<Row, Row> streamingItr =
+ testStreamingRead(
+ buildSimpleQuery(table),
+ Arrays.asList(
+ // part = 2022-01-01
+ changelogRow("+I", "US Dollar", 120L, "2022-01-01"),
+ // part = 2022-01-02
+ changelogRow("+I", "Euro", 119L, "2022-01-02")));
+
+ // test refresh after overwriting
+ insertOverwritePartition(
+ table, "PARTITION (dt = '2022-01-02')", "('Euro', 100)", "('Yen', 1)");
+
+ validateStreamingReadResult(
+ streamingItr,
+ Arrays.asList(
+ // part = 2022-01-02
+ changelogRow("-D", "Euro", 119L, "2022-01-02"),
+ changelogRow("+I", "Euro", 100L, "2022-01-02"),
+ changelogRow("+I", "Yen", 1L, "2022-01-02")));
+
+ streamingItr.close();
+ }
+
+ @Test
+ public void testStreamingReadOverwriteWithoutPartitionedRecords() throws Exception {
+ String table =
+ createTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
+ Collections.singletonList("currency"),
+ Collections.emptyList(),
+ streamingReadOverwrite);
+
+ insertInto(
+ table,
+ "('US Dollar', 102, '2022-01-01')",
+ "('Yen', 1, '2022-01-02')",
+ "('Euro', 119, '2022-01-02')");
+
+ checkFileStorePath(table, Collections.emptyList());
+
+ // test projection and filter
+ BlockingIterator<Row, Row> streamingItr =
+ testStreamingRead(
+ buildQuery(table, "currency, rate", "WHERE dt = '2022-01-02'"),
+ Arrays.asList(
+ changelogRow("+I", "Yen", 1L), changelogRow("+I", "Euro", 119L)));
+
+ insertOverwrite(table, "('US Dollar', 100, '2022-01-02')", "('Yen', 10, '2022-01-01')");
+
+ validateStreamingReadResult(
+ streamingItr,
+ Arrays.asList(
+ changelogRow("-D", "Yen", 1L),
+ changelogRow("-D", "Euro", 119L),
+ changelogRow("+I", "US Dollar", 100L)));
+
+ streamingItr.close();
+ }
+
+ @Test
+ public void testUnsupportStreamingReadOverwriteWithoutPk() {
+ assertThatThrownBy(
+ () ->
+ createTable(
+ Arrays.asList(
+ "currency STRING", "rate BIGINT", "dt String"),
+ Collections.emptyList(),
+ Collections.singletonList("dt"),
+ streamingReadOverwrite))
+ .satisfies(
+ FlinkAssertions.anyCauseMatches(
+ RuntimeException.class,
+ "Doesn't support streaming read the changes from overwrite when the primary keys are not defined."));
+ }
+
// ----------------------------------------------------------------------------------------------------------------
// Keyword
// ----------------------------------------------------------------------------------------------------------------
@@ -782,7 +859,8 @@ public class ReadWriteTableITCase extends AbstractTestBase {
changelogRow("+I", 2, "test%2"),
changelogRow("+I", 8, "tests"),
changelogRow("+I", 10, "tested"),
- changelogRow("+I", 20, "test_123")));
+ changelogRow("+I", 20, "test_123"),
+ changelogRow("+I", 100, "test%fff")));
testBatchRead(
buildQuery(table, "*", "WHERE f1 LIKE 'v%'"),
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
index 7061f087..677c15eb 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
@@ -144,6 +144,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// check no changelog generated for streaming read
assertNoMoreRecords(streamItr);
+ streamItr.close();
// batch read to check data refresh
testBatchRead(
@@ -156,8 +157,6 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
changelogRow("+I", "Yen", 20L, "2022-01-03"),
changelogRow("+I", "Yen", 20L, "2022-01-04")));
- streamItr.close();
-
// filter on partition
testStreamingRead(
buildQuery(table, "*", "WHERE dt = '2022-01-01'"),
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
index 4dd94b51..22279f28 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
@@ -42,6 +42,8 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeoutException;
/** IT cases for {@link CompactAction}. */
public class CompactActionITCase extends ActionITCaseBase {
@@ -53,7 +55,7 @@ public class CompactActionITCase extends ActionITCaseBase {
RowType.of(FIELD_TYPES, new String[] {"k", "v", "hh", "dt"});
@Test
- @Timeout(60000)
+ @Timeout(60)
public void testBatchCompact() throws Exception {
Map<String, String> options = new HashMap<>();
options.put(CoreOptions.WRITE_ONLY.key(), "true");
@@ -84,7 +86,7 @@ public class CompactActionITCase extends ActionITCaseBase {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRuntimeMode(RuntimeExecutionMode.BATCH);
- env.setParallelism(2);
+ env.setParallelism(ThreadLocalRandom.current().nextInt(2) + 1);
new CompactAction(tablePath).withPartitions(getSpecifiedPartitions()).build(env);
env.execute();
@@ -106,7 +108,6 @@ public class CompactActionITCase extends ActionITCaseBase {
}
@Test
- @Timeout(60000)
public void testStreamingCompact() throws Exception {
Map<String, String> options = new HashMap<>();
options.put(CoreOptions.CHANGELOG_PRODUCER.key(), "full-compaction");
@@ -148,27 +149,16 @@ public class CompactActionITCase extends ActionITCaseBase {
env.setRuntimeMode(RuntimeExecutionMode.STREAMING);
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
env.getCheckpointConfig().setCheckpointInterval(500);
- env.setParallelism(2);
+ env.setParallelism(ThreadLocalRandom.current().nextInt(2) + 1);
new CompactAction(tablePath).withPartitions(getSpecifiedPartitions()).build(env);
env.executeAsync();
- while (true) {
- plan = snapshotEnumerator.enumerate();
- if (plan != null) {
- break;
- }
- Thread.sleep(1000);
- }
-
// first full compaction
- List<String> actual = new ArrayList<>();
- while (plan != null) {
- actual.addAll(getResult(table.newRead(), plan.splits(), ROW_TYPE));
- plan = snapshotEnumerator.enumerate();
- }
- actual.sort(String::compareTo);
- Assertions.assertEquals(
- Arrays.asList("+I[1, 100, 15, 20221208]", "+I[1, 100, 15, 20221209]"), actual);
+ validateResult(
+ table,
+ snapshotEnumerator,
+ Arrays.asList("+I[1, 100, 15, 20221208]", "+I[1, 100, 15, 20221209]"),
+ 60_000);
// incremental records
writeData(
@@ -176,28 +166,16 @@ public class CompactActionITCase extends ActionITCaseBase {
rowData(1, 101, 16, BinaryString.fromString("20221208")),
rowData(1, 101, 15, BinaryString.fromString("20221209")));
- while (true) {
- plan = snapshotEnumerator.enumerate();
- if (plan != null) {
- break;
- }
- Thread.sleep(1000);
- }
-
// second full compaction
- actual = new ArrayList<>();
- while (plan != null) {
- actual.addAll(getResult(table.newRead(), plan.splits(), ROW_TYPE));
- plan = snapshotEnumerator.enumerate();
- }
- actual.sort(String::compareTo);
- Assertions.assertEquals(
+ validateResult(
+ table,
+ snapshotEnumerator,
Arrays.asList(
"+U[1, 101, 15, 20221208]",
"+U[1, 101, 15, 20221209]",
"-U[1, 100, 15, 20221208]",
"-U[1, 100, 15, 20221209]"),
- actual);
+ 60_000);
// assert dedicated compact job will expire snapshots
Assertions.assertEquals(
@@ -216,4 +194,31 @@ public class CompactActionITCase extends ActionITCaseBase {
return Arrays.asList(partition1, partition2);
}
+
+ private void validateResult(
+ FileStoreTable table,
+ SnapshotEnumerator snapshotEnumerator,
+ List<String> expected,
+ long timeout)
+ throws Exception {
+ List<String> actual = new ArrayList<>();
+ long start = System.currentTimeMillis();
+ while (actual.size() != expected.size()) {
+ DataTableScan.DataFilePlan plan = snapshotEnumerator.enumerate();
+ if (plan != null) {
+ actual.addAll(getResult(table.newRead(), plan.splits(), ROW_TYPE));
+ }
+ if (System.currentTimeMillis() - start > timeout) {
+ break;
+ }
+ }
+ if (actual.size() != expected.size()) {
+ throw new TimeoutException(
+ String.format(
+ "Cannot collect %s records in %s milliseconds.",
+ expected.size(), timeout));
+ }
+ actual.sort(String::compareTo);
+ Assertions.assertEquals(expected, actual);
+ }
}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
index 8d7b1937..be106268 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
@@ -28,7 +28,6 @@ import org.apache.flink.table.store.table.source.DataSplit;
import org.apache.flink.table.store.table.source.DataTableScan;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
import javax.annotation.Nullable;
@@ -44,8 +43,6 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link FileStoreSourceSplitGenerator}. */
public class FileStoreSourceSplitGeneratorTest {
- @TempDir java.nio.file.Path tempDir;
-
@Test
public void test() {
FileStoreScan.Plan plan =
@@ -80,8 +77,9 @@ public class FileStoreSourceSplitGeneratorTest {
AbstractDataTableScan.generateSplits(
1L,
false,
+ false,
Collections::singletonList,
- plan.groupByPartFiles(plan.files(FileKind.ADD)));
+ FileStoreScan.Plan.groupByPartFiles(plan.files(FileKind.ADD)));
DataTableScan.DataFilePlan tableScanPlan = new DataTableScan.DataFilePlan(1L, scanSplits);
List<FileStoreSourceSplit> splits =
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
index 709f6594..f16c90af 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
@@ -225,54 +225,47 @@ public class ReadWriteTableTestUtil {
public static void testBatchRead(String query, List<Row> expected) throws Exception {
CloseableIterator<Row> resultItr = bEnv.executeSql(query).collect();
try (BlockingIterator<Row, Row> iterator = BlockingIterator.of(resultItr)) {
- if (expected.isEmpty()) {
- assertThat(resultItr.hasNext()).isFalse();
- } else {
+ if (!expected.isEmpty()) {
assertThat(
iterator.collect(
expected.size(), TIME_OUT.getSize(), TIME_OUT.getUnit()))
.containsExactlyInAnyOrderElementsOf(expected);
}
+ assertThat(resultItr.hasNext()).isFalse();
}
}
public static BlockingIterator<Row, Row> testStreamingRead(String query, List<Row> expected)
throws Exception {
BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
-
- if (expected.isEmpty()) {
- assertNoMoreRecords(iterator);
- } else {
- assertThat(iterator.collect(expected.size()))
- .containsExactlyInAnyOrderElementsOf(expected);
- }
-
+ validateStreamingReadResult(iterator, expected);
return iterator;
}
public static BlockingIterator<Row, Row> testStreamingReadWithReadFirst(
String source, String sink, String query, List<Row> expected) throws Exception {
BlockingIterator<Row, Row> iterator = BlockingIterator.of(sEnv.executeSql(query).collect());
-
insertIntoFromTable(source, sink);
+ validateStreamingReadResult(iterator, expected);
+ return iterator;
+ }
+ public static void validateStreamingReadResult(
+ BlockingIterator<Row, Row> streamingItr, List<Row> expected) throws Exception {
if (expected.isEmpty()) {
- assertNoMoreRecords(iterator);
+ assertNoMoreRecords(streamingItr);
} else {
- assertThat(iterator.collect(expected.size()))
+ assertThat(streamingItr.collect(expected.size()))
.containsExactlyInAnyOrderElementsOf(expected);
}
-
- return iterator;
}
- public static void assertNoMoreRecords(BlockingIterator<Row, Row> iterator) throws Exception {
+ public static void assertNoMoreRecords(BlockingIterator<Row, Row> iterator) {
List<Row> expectedRecords = Collections.emptyList();
try {
// set expectation size to 1 to let time pass by until timeout
// just wait 5s to avoid too long time
expectedRecords = iterator.collect(1, 5L, TimeUnit.SECONDS);
- iterator.close();
} catch (TimeoutException ignored) {
// don't throw exception
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
index 84e019a6..f31e60c1 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
@@ -401,6 +401,13 @@ public class CoreOptions implements Serializable {
.withDescription(
"Whether to create underlying storage when reading and writing the table.");
+ public static final ConfigOption<Boolean> STREAMING_READ_OVERWRITE =
+ ConfigOptions.key("streaming-read-overwrite")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Whether to read the changes from overwrite in streaming mode.");
+
private final Configuration options;
public CoreOptions(Map<String, String> options) {
@@ -604,6 +611,10 @@ public class CoreOptions implements Serializable {
return options.get(WRITE_ONLY);
}
+ public boolean streamingReadOverwrite() {
+ return options.get(STREAMING_READ_OVERWRITE);
+ }
+
/** Specifies the merge engine for table with primary key. */
public enum MergeEngine implements DescribedEnum {
DEDUPLICATE("deduplicate", "De-duplicate and keep the last row."),
@@ -841,7 +852,7 @@ public class CoreOptions implements Serializable {
!SYSTEM_FIELD_NAMES.contains(f),
String.format(
"Field name[%s] in schema cannot be exist in [%s]",
- f, SYSTEM_FIELD_NAMES.toString()));
+ f, SYSTEM_FIELD_NAMES));
checkState(
!f.startsWith(KEY_FIELD_PREFIX),
String.format(
@@ -855,6 +866,11 @@ public class CoreOptions implements Serializable {
"Cannot define any primary key in an append-only table. Set 'write-mode'='change-log' if "
+ "still want to keep the primary key definition.");
}
+
+ if (schema.primaryKeys().isEmpty() && options.streamingReadOverwrite()) {
+ throw new RuntimeException(
+ "Doesn't support streaming read the changes from overwrite when the primary keys are not defined.");
+ }
}
private static void checkOptionExistInMode(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/KeyValue.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/KeyValue.java
index 556547dc..e979bd9e 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/KeyValue.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/KeyValue.java
@@ -73,6 +73,11 @@ public class KeyValue {
return this;
}
+ public KeyValue replaceValueKind(RowKind valueKind) {
+ this.valueKind = valueKind;
+ return this;
+ }
+
public InternalRow key() {
return key;
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
index 5ed517a0..77fc5f31 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
@@ -72,7 +72,7 @@ public interface FileStoreScan {
}
/** Return a map group by partition and bucket. */
- default Map<BinaryRow, Map<Integer, List<DataFileMeta>>> groupByPartFiles(
+ static Map<BinaryRow, Map<Integer, List<DataFileMeta>>> groupByPartFiles(
List<ManifestEntry> files) {
Map<BinaryRow, Map<Integer, List<DataFileMeta>>> groupBy = new HashMap<>();
for (ManifestEntry entry : files) {
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/KeyValueFileStoreRead.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/KeyValueFileStoreRead.java
index 419f6b45..af4adcba 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/KeyValueFileStoreRead.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/KeyValueFileStoreRead.java
@@ -169,7 +169,10 @@ public class KeyValueFileStoreRead implements FileStoreRead<KeyValue> {
file.schemaId(), fileName, file.level());
});
}
- return ConcatRecordReader.create(suppliers);
+ RecordReader<KeyValue> concatRecordReader = ConcatRecordReader.create(suppliers);
+ return split.reverseRowKind()
+ ? new ReverseReader(concatRecordReader)
+ : concatRecordReader;
} else {
// Sections are read by SortMergeReader, which sorts and merges records by keys.
// So we cannot project keys or else the sorting will be incorrect.
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/ReverseReader.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/ReverseReader.java
new file mode 100644
index 00000000..752aed4e
--- /dev/null
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/ReverseReader.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.operation;
+
+import org.apache.flink.table.store.file.KeyValue;
+import org.apache.flink.table.store.file.utils.RecordReader;
+import org.apache.flink.table.store.types.RowKind;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+
+/**
+ * A {@link RecordReader} which reverses the {@link RowKind} of the {@link KeyValue} from {@link
+ * RowKind#INSERT} to {@link RowKind#DELETE} read from the wrapped reader.
+ */
+public class ReverseReader implements RecordReader<KeyValue> {
+
+ private final RecordReader<KeyValue> reader;
+
+ public ReverseReader(RecordReader<KeyValue> reader) {
+ this.reader = reader;
+ }
+
+ @Nullable
+ @Override
+ public RecordIterator<KeyValue> readBatch() throws IOException {
+ RecordIterator<KeyValue> batch = reader.readBatch();
+
+ if (batch == null) {
+ return null;
+ }
+
+ return new RecordIterator<KeyValue>() {
+ @Override
+ public KeyValue next() throws IOException {
+ KeyValue kv = batch.next();
+ if (kv == null) {
+ return null;
+ }
+ if (kv.valueKind() == RowKind.UPDATE_BEFORE || kv.valueKind() == RowKind.DELETE) {
+ throw new IllegalStateException(
+ "In reverse reader, the value kind of records cannot be UPDATE_BEFORE or DELETE.");
+ }
+
+ return kv.replaceValueKind(RowKind.DELETE);
+ }
+
+ @Override
+ public void releaseBatch() {
+ batch.releaseBatch();
+ }
+ };
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTable.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTable.java
index f97b7550..2540f2e9 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTable.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTable.java
@@ -24,6 +24,7 @@ import org.apache.flink.table.store.file.AppendOnlyFileStore;
import org.apache.flink.table.store.file.WriteMode;
import org.apache.flink.table.store.file.operation.AppendOnlyFileStoreRead;
import org.apache.flink.table.store.file.operation.AppendOnlyFileStoreScan;
+import org.apache.flink.table.store.file.operation.ReverseReader;
import org.apache.flink.table.store.file.predicate.Predicate;
import org.apache.flink.table.store.file.schema.TableSchema;
import org.apache.flink.table.store.file.utils.FileStorePathFactory;
@@ -81,6 +82,18 @@ public class AppendOnlyFileStoreTable extends AbstractFileStoreTable {
AppendOnlyFileStoreScan scan = store().newScan();
return new AbstractDataTableScan(
fileIO, scan, tableSchema, store().pathFactory(), options()) {
+ /**
+ * Currently, the streaming read of overwrite is implemented by reversing the {@link
+ * RowKind} of overwrote records to {@link RowKind#DELETE}, so only tables that have
+ * primary key support it.
+ *
+ * @see ReverseReader
+ */
+ @Override
+ public boolean supportStreamingReadOverwrite() {
+ return false;
+ }
+
@Override
protected SplitGenerator splitGenerator(FileStorePathFactory pathFactory) {
return new AppendOnlySplitGenerator(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogValueCountFileStoreTable.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogValueCountFileStoreTable.java
index f20d9cbe..8181e249 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogValueCountFileStoreTable.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogValueCountFileStoreTable.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.store.file.WriteMode;
import org.apache.flink.table.store.file.io.DataFileMeta;
import org.apache.flink.table.store.file.mergetree.compact.ValueCountMergeFunction;
import org.apache.flink.table.store.file.operation.KeyValueFileStoreScan;
+import org.apache.flink.table.store.file.operation.ReverseReader;
import org.apache.flink.table.store.file.predicate.Predicate;
import org.apache.flink.table.store.file.schema.KeyValueFieldsExtractor;
import org.apache.flink.table.store.file.schema.TableSchema;
@@ -96,6 +97,18 @@ public class ChangelogValueCountFileStoreTable extends AbstractFileStoreTable {
KeyValueFileStoreScan scan = store().newScan();
return new AbstractDataTableScan(
fileIO, scan, tableSchema, store().pathFactory(), options()) {
+ /**
+ * Currently, the streaming read of overwrite is implemented by reversing the {@link
+ * RowKind} of overwrote records to {@link RowKind#DELETE}, so only tables that have
+ * primary key support it.
+ *
+ * @see ReverseReader
+ */
+ @Override
+ public boolean supportStreamingReadOverwrite() {
+ return false;
+ }
+
@Override
protected SplitGenerator splitGenerator(FileStorePathFactory pathFactory) {
return new MergeTreeSplitGenerator(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogWithKeyFileStoreTable.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogWithKeyFileStoreTable.java
index b7817133..00a68d60 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogWithKeyFileStoreTable.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/ChangelogWithKeyFileStoreTable.java
@@ -152,6 +152,11 @@ public class ChangelogWithKeyFileStoreTable extends AbstractFileStoreTable {
KeyValueFileStoreScan scan = store().newScan();
return new AbstractDataTableScan(
fileIO(), scan, tableSchema, store().pathFactory(), options()) {
+ @Override
+ public boolean supportStreamingReadOverwrite() {
+ return new CoreOptions(tableSchema.options()).streamingReadOverwrite();
+ }
+
@Override
protected SplitGenerator splitGenerator(FileStorePathFactory pathFactory) {
return new MergeTreeSplitGenerator(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
index 632787c0..4c020518 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
@@ -126,8 +126,43 @@ public abstract class AbstractDataTableScan implements DataTableScan {
generateSplits(
snapshotId == null ? Snapshot.FIRST_SNAPSHOT_ID - 1 : snapshotId,
scanKind != ScanKind.ALL,
+ false,
splitGenerator(pathFactory),
- plan.groupByPartFiles(plan.files(FileKind.ADD)));
+ FileStoreScan.Plan.groupByPartFiles(plan.files(FileKind.ADD)));
+ return new DataFilePlan(snapshotId, splits);
+ }
+
+ @Override
+ public DataFilePlan planOverwriteChanges() {
+ withKind(ScanKind.DELTA);
+ FileStoreScan.Plan plan = scan.plan();
+ long snapshotId = plan.snapshotId();
+
+ SnapshotManager snapshotManager = new SnapshotManager(fileIO, pathFactory.root());
+ Snapshot snapshot = snapshotManager.snapshot(snapshotId);
+ if (snapshot.commitKind() != Snapshot.CommitKind.OVERWRITE) {
+ throw new IllegalStateException(
+ "Cannot plan overwrite changes from a non-overwrite snapshot.");
+ }
+
+ List<DataSplit> splits = new ArrayList<>();
+
+ splits.addAll(
+ generateSplits(
+ snapshotId,
+ true,
+ true,
+ splitGenerator(pathFactory),
+ FileStoreScan.Plan.groupByPartFiles(plan.files(FileKind.DELETE))));
+
+ splits.addAll(
+ generateSplits(
+ snapshotId,
+ true,
+ false,
+ splitGenerator(pathFactory),
+ FileStoreScan.Plan.groupByPartFiles(plan.files(FileKind.ADD))));
+
return new DataFilePlan(snapshotId, splits);
}
@@ -135,6 +170,7 @@ public abstract class AbstractDataTableScan implements DataTableScan {
public static List<DataSplit> generateSplits(
long snapshotId,
boolean isIncremental,
+ boolean reverseRowKind,
SplitGenerator splitGenerator,
Map<BinaryRow, Map<Integer, List<DataFileMeta>>> groupedDataFiles) {
List<DataSplit> splits = new ArrayList<>();
@@ -148,13 +184,23 @@ public abstract class AbstractDataTableScan implements DataTableScan {
// Don't split when incremental
splits.add(
new DataSplit(
- snapshotId, partition, bucket, bucketEntry.getValue(), true));
+ snapshotId,
+ partition,
+ bucket,
+ bucketEntry.getValue(),
+ true,
+ reverseRowKind));
} else {
splitGenerator.split(bucketEntry.getValue()).stream()
.map(
files ->
new DataSplit(
- snapshotId, partition, bucket, files, false))
+ snapshotId,
+ partition,
+ bucket,
+ files,
+ false,
+ reverseRowKind))
.forEach(splits::add);
}
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataSplit.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataSplit.java
index 37e5c592..7e354c66 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataSplit.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataSplit.java
@@ -45,13 +45,27 @@ public class DataSplit implements Split {
private List<DataFileMeta> files;
private boolean isIncremental;
+ // when reverseRowKind is true, the RowKind of records from this split should be reversed to
+ // DELETE
+ private boolean reverseRowKind;
+
public DataSplit(
long snapshotId,
BinaryRow partition,
int bucket,
List<DataFileMeta> files,
boolean isIncremental) {
- init(snapshotId, partition, bucket, files, isIncremental);
+ init(snapshotId, partition, bucket, files, isIncremental, false);
+ }
+
+ public DataSplit(
+ long snapshotId,
+ BinaryRow partition,
+ int bucket,
+ List<DataFileMeta> files,
+ boolean isIncremental,
+ boolean reverseRowKind) {
+ init(snapshotId, partition, bucket, files, isIncremental, reverseRowKind);
}
private void init(
@@ -59,12 +73,14 @@ public class DataSplit implements Split {
BinaryRow partition,
int bucket,
List<DataFileMeta> files,
- boolean isIncremental) {
+ boolean isIncremental,
+ boolean reverseRowKind) {
this.snapshotId = snapshotId;
this.partition = partition;
this.bucket = bucket;
this.files = files;
this.isIncremental = isIncremental;
+ this.reverseRowKind = reverseRowKind;
}
public long snapshotId() {
@@ -87,6 +103,10 @@ public class DataSplit implements Split {
return isIncremental;
}
+ public boolean reverseRowKind() {
+ return reverseRowKind;
+ }
+
@Override
public long rowCount() {
long rowCount = 0;
@@ -108,12 +128,13 @@ public class DataSplit implements Split {
return bucket == split.bucket
&& Objects.equals(partition, split.partition)
&& Objects.equals(files, split.files)
- && isIncremental == split.isIncremental;
+ && isIncremental == split.isIncremental
+ && reverseRowKind == split.reverseRowKind;
}
@Override
public int hashCode() {
- return Objects.hash(partition, bucket, files, isIncremental);
+ return Objects.hash(partition, bucket, files, isIncremental, reverseRowKind);
}
private void writeObject(ObjectOutputStream out) throws IOException {
@@ -122,7 +143,13 @@ public class DataSplit implements Split {
private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
DataSplit split = DataSplit.deserialize(new DataInputViewStreamWrapper(in));
- init(split.snapshotId, split.partition, split.bucket, split.files, split.isIncremental);
+ init(
+ split.snapshotId,
+ split.partition,
+ split.bucket,
+ split.files,
+ split.isIncremental,
+ split.reverseRowKind);
}
public void serialize(DataOutputView out) throws IOException {
@@ -135,6 +162,7 @@ public class DataSplit implements Split {
dataFileSer.serialize(file, out);
}
out.writeBoolean(isIncremental);
+ out.writeBoolean(reverseRowKind);
}
public static DataSplit deserialize(DataInputView in) throws IOException {
@@ -147,6 +175,7 @@ public class DataSplit implements Split {
for (int i = 0; i < fileNumber; i++) {
files.add(dataFileSer.deserialize(in));
}
- return new DataSplit(snapshotId, partition, bucket, files, in.readBoolean());
+ return new DataSplit(
+ snapshotId, partition, bucket, files, in.readBoolean(), in.readBoolean());
}
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataTableScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataTableScan.java
index 5509929f..cfcc0f9f 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataTableScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/DataTableScan.java
@@ -40,6 +40,11 @@ public interface DataTableScan extends TableScan {
DataTableScan.DataFilePlan plan();
+ /** to extract plan from overwrite changes. */
+ DataTableScan.DataFilePlan planOverwriteChanges();
+
+ boolean supportStreamingReadOverwrite();
+
/** Scanning plan containing snapshot ID and input splits. */
class DataFilePlan implements Plan {
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/snapshot/ContinuousDataFileSnapshotEnumerator.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/snapshot/ContinuousDataFileSnapshotEnumerator.java
index 8fbf244d..41d3811a 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/snapshot/ContinuousDataFileSnapshotEnumerator.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/snapshot/ContinuousDataFileSnapshotEnumerator.java
@@ -95,7 +95,15 @@ public class ContinuousDataFileSnapshotEnumerator implements SnapshotEnumerator
Snapshot snapshot = snapshotManager.snapshot(nextSnapshotId);
- if (followUpScanner.shouldScanSnapshot(snapshot)) {
+ // first check changes of overwrite
+ if (snapshot.commitKind() == Snapshot.CommitKind.OVERWRITE
+ && scan.supportStreamingReadOverwrite()) {
+ LOG.debug("Find overwrite snapshot id {}.", nextSnapshotId);
+ DataTableScan.DataFilePlan overwritePlan =
+ scan.withSnapshot(nextSnapshotId).planOverwriteChanges();
+ nextSnapshotId++;
+ return overwritePlan;
+ } else if (followUpScanner.shouldScanSnapshot(snapshot)) {
LOG.debug("Find snapshot id {}.", nextSnapshotId);
DataTableScan.DataFilePlan plan = followUpScanner.getPlan(nextSnapshotId, scan);
nextSnapshotId++;
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/system/AuditLogTable.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/system/AuditLogTable.java
index f75f6ad0..6849269d 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/system/AuditLogTable.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/system/AuditLogTable.java
@@ -180,6 +180,16 @@ public class AuditLogTable implements DataTable {
public DataTableScan.DataFilePlan plan() {
return dataScan.plan();
}
+
+ @Override
+ public DataFilePlan planOverwriteChanges() {
+ return dataScan.planOverwriteChanges();
+ }
+
+ @Override
+ public boolean supportStreamingReadOverwrite() {
+ return dataScan.supportStreamingReadOverwrite();
+ }
}
private class AuditLogRead implements TableRead {