You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2023/02/09 05:36:12 UTC
[flink-table-store] branch master updated: [FLINK-30918] Refactor CompositePkAndMultiPartitionedTableITCase to get rid of managed table
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
The following commit(s) were added to refs/heads/master by this push:
new ac01ddc3 [FLINK-30918] Refactor CompositePkAndMultiPartitionedTableITCase to get rid of managed table
ac01ddc3 is described below
commit ac01ddc333372dc92bf9bc3080b7dd4d90d26d01
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Thu Feb 9 13:36:07 2023 +0800
[FLINK-30918] Refactor CompositePkAndMultiPartitionedTableITCase to get rid of managed table
This closes #505
---
.../CompositePkAndMultiPartitionedTableITCase.java | 1339 ++++++--------------
...AndMultiPartitionedTableWIthKafkaLogITCase.java | 1134 +++++++++++++++++
.../store/connector/ReadWriteTableTestBase.java | 203 ---
.../table/store/connector/ShowCreateUtil.java | 40 -
.../StreamingReadWriteTableWithKafkaLogITCase.java | 252 ++--
.../connector/action/CompactActionITCase.java | 12 +-
.../connector/util/ReadWriteTableTestUtil.java | 41 +
.../table/store/kafka/KafkaTableTestBase.java | 6 +-
8 files changed, 1654 insertions(+), 1373 deletions(-)
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
index a0a6b3f6..693adce9 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
@@ -18,48 +18,46 @@
package org.apache.flink.table.store.connector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.table.api.EnvironmentSettings;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.table.store.file.utils.BlockingIterator;
+import org.apache.flink.table.store.connector.util.AbstractTestBase;
import org.apache.flink.types.Row;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyExchangeRates;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyExchangeRatesChangelogWithoutUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithoutUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.exchangeRatesChangelogWithoutUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyExchangeRates;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyExchangeRatesChangelogWithoutUB;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyRates;
-import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyRatesChangelogWithoutUB;
-import static org.assertj.core.api.Assertions.assertThat;
-
-/**
- * Table store IT case when the managed table has composite primary keys and multiple partition
- * fields.
- */
-public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTestBase {
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.bEnv;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQuery;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildSimpleQuery;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.checkFileStorePath;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTemporaryTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.init;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertIntoFromTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertOverwrite;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertOverwritePartition;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testBatchRead;
+
+/** Table store IT case when the table has composite primary keys and multiple partition fields. */
+public class CompositePkAndMultiPartitionedTableITCase extends AbstractTestBase {
+
+ @BeforeEach
+ public void setUp() {
+ init(getTempDirPath());
+ }
@Test
public void testBatchWriteWithMultiPartitionedRecordsWithMultiPk() throws Exception {
- // input is hourlyExchangeRates()
- List<Row> expectedRecords =
+ List<Row> initialRecords =
Arrays.asList(
// to_currency is USD, dt = 2022-01-01, hh = 11
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01", "11"),
changelogRow(
"+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01", "11"),
changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "11"),
@@ -68,69 +66,83 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "12"),
changelogRow(
"+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12"),
// to_currency is Euro, dt = 2022-01-02, hh = 23
changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23"));
- // test batch read
- String managedTable =
- collectAndCheckBatchReadWrite(
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
Arrays.asList("dt", "hh"),
+ initialRecords,
+ "dt:2022-01-01,hh:11;dt:2022-01-01,hh:12;dt:2022-01-02,hh:23",
+ true,
+ "I");
+
+ String table =
+ createTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
Arrays.asList("from_currency", "to_currency", "dt", "hh"),
- null,
- Collections.emptyList(),
- expectedRecords);
-
- checkFileStorePath(tEnv, managedTable, hourlyExchangeRates().f1);
-
- // test streaming read
- final StreamTableEnvironment streamTableEnv =
- StreamTableEnvironment.create(buildStreamEnv());
- registerTable(streamTableEnv, managedTable);
- BlockingIterator<Row, Row> streamIter =
- collectAndCheck(
- streamTableEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- expectedRecords);
+ Arrays.asList("dt", "hh"));
- // overwrite static partition dt = 2022-01-02 and hh = 23
- Map<String, String> overwritePartition = new LinkedHashMap<>();
- overwritePartition.put("dt", "'2022-01-02'");
- overwritePartition.put("hh", "'23'");
- prepareEnvAndOverwrite(
- managedTable,
- overwritePartition,
- Collections.singletonList(new String[] {"'US Dollar'", "'Thai Baht'", "33.51"}));
+ insertIntoFromTable(temporaryTable, table);
- // streaming iter will not receive any changelog
- assertNoMoreRecords(streamIter);
+ checkFileStorePath(
+ table,
+ Arrays.asList("dt=2022-01-01,hh=11", "dt=2022-01-01,hh=12", "dt=2022-01-02,hh=23"));
+
+ // test batch read
+ testBatchRead(
+ buildSimpleQuery(table),
+ Arrays.asList(
+ // to_currency is USD, dt = 2022-01-01, hh = 11
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01", "11"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "11"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
+ // to_currency is USD, dt = 2022-01-01, hh = 12
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
+ changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
+ changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12"),
+ // to_currency is Euro, dt = 2022-01-02, hh = 23
+ changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")));
+
+ // overwrite static partition dt = 2022-01-02 and hh = 23
+ insertOverwritePartition(
+ table,
+ "PARTITION (dt = '2022-01-02', hh = '23')",
+ "('US Dollar', 'Thai Baht', 33.51)");
// batch read to check partition refresh
- collectAndCheck(
- tEnv,
- managedTable,
- Collections.emptyMap(),
- "dt = '2022-01-02' AND hh = '23'",
- Collections.singletonList(
- changelogRow(
- "+I",
- "US Dollar",
- "Thai Baht",
- 33.51d,
- "2022-01-02",
- "23")))
- .close();
+ testBatchRead(
+ buildQuery(table, "*", "WHERE dt = '2022-01-02' AND hh = '23'"),
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-02", "23")));
// test partition filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "dt = '2022-01-01'",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE dt = '2022-01-01'"),
Arrays.asList(
// to_currency is USD, dt = 2022-01-01, hh = 11
changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
@@ -147,11 +159,8 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
"+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12")));
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "dt = '2022-01-01' AND hh = '12'",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE dt = '2022-01-01' AND hh = '12'"),
Arrays.asList(
changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
@@ -161,29 +170,17 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12")));
// test field filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "to_currency = 'Euro'",
- Collections.emptyList(),
- Arrays.asList(
- changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
- changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")));
+ testBatchRead(
+ buildQuery(table, "*", "WHERE to_currency = 'Euro'"), Collections.emptyList());
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "from_currency = 'HK Dollar'",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE from_currency = 'HK Dollar'"),
Arrays.asList(
changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12")));
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "rate_by_to_currency > 0.5",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE rate_by_to_currency > 0.5"),
Arrays.asList(
changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
changelogRow(
@@ -193,142 +190,158 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
changelogRow(
"+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
- changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
- changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")));
+ changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-02", "23")));
// test partition and field filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "rate_by_to_currency > 0.9 AND hh = '23'",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE rate_by_to_currency > 1 AND hh = '12'"),
Collections.singletonList(
- changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23")));
+ changelogRow("+I", "Euro", "US Dollar", 1.12, "2022-01-01", "12")));
// test projection
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- null,
- Arrays.asList("from_currency", "dt", "hh"),
+ testBatchRead(
+ buildQuery(table, "from_currency, dt, hh", ""),
Arrays.asList(
- // to_currency is USD, dt = 2022-01-01, hh = 11
+ // dt = 2022-01-01, hh = 11
changelogRow("+I", "Euro", "2022-01-01", "11"),
changelogRow("+I", "HK Dollar", "2022-01-01", "11"),
changelogRow("+I", "Singapore Dollar", "2022-01-01", "11"),
changelogRow("+I", "Yen", "2022-01-01", "11"),
changelogRow("+I", "US Dollar", "2022-01-01", "11"),
- // to_currency is USD, dt = 2022-01-01, hh = 12
+ // dt = 2022-01-01, hh = 12
changelogRow("+I", "US Dollar", "2022-01-01", "12"),
changelogRow("+I", "Euro", "2022-01-01", "12"),
changelogRow("+I", "HK Dollar", "2022-01-01", "12"),
changelogRow("+I", "Singapore Dollar", "2022-01-01", "12"),
changelogRow("+I", "Yen", "2022-01-01", "12"),
- // to_currency is Euro, dt = 2022-01-02, hh = 23
- changelogRow("+I", "US Dollar", "2022-01-02", "23"),
- changelogRow("+I", "Singapore Dollar", "2022-01-02", "23")));
+ // dt = 2022-01-02, hh = 23
+ changelogRow("+I", "US Dollar", "2022-01-02", "23")));
// test projection and filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "dt = '2022-01-01' AND hh >= '12' OR rate_by_to_currency > 2",
- Arrays.asList("from_currency", "to_currency"),
+ testBatchRead(
+ buildQuery(
+ table,
+ "from_currency, to_currency",
+ "WHERE dt = '2022-01-01' AND hh >= '12' OR rate_by_to_currency > 2"),
Arrays.asList(
- // to_currency is USD, dt = 2022-01-01, hh = 12
+ // dt = 2022-01-01, hh = 12
changelogRow("+I", "US Dollar", "US Dollar"),
changelogRow("+I", "Euro", "US Dollar"),
changelogRow("+I", "HK Dollar", "US Dollar"),
changelogRow("+I", "Singapore Dollar", "US Dollar"),
- changelogRow("+I", "Yen", "US Dollar")));
+ changelogRow("+I", "Yen", "US Dollar"),
+ // dt = 2022-01-02, hh = 23
+ changelogRow("+I", "US Dollar", "Thai Baht")));
}
@Test
public void testBatchWriteWithSinglePartitionedRecordsWithMultiPk() throws Exception {
- // input is dailyExchangeRates()
- List<Row> expectedRecords =
+ List<Row> initialRecords =
Arrays.asList(
+ // to_currency is USD
changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02"),
+ // to_currency is Euro
changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-01"),
changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-01"),
+ // to_currency is Yen
changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-01"),
changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-01"),
- changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01"),
- changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
- changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02"));
- // test batch read
- String managedTable =
- collectAndCheckBatchReadWrite(
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01"));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
Collections.singletonList("dt"),
+ initialRecords,
+ "dt:2022-01-01;dt:2022-01-02",
+ true,
+ "I");
+
+ String table =
+ createTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
Arrays.asList("from_currency", "to_currency", "dt"),
- null,
- Collections.emptyList(),
- expectedRecords);
-
- checkFileStorePath(tEnv, managedTable, dailyExchangeRates().f1);
-
- // test streaming read
- final StreamTableEnvironment streamTableEnv =
- StreamTableEnvironment.create(buildStreamEnv());
- registerTable(streamTableEnv, managedTable);
- BlockingIterator<Row, Row> streamIter =
- collectAndCheck(
- streamTableEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- expectedRecords);
+ Collections.singletonList("dt"));
+
+ insertIntoFromTable(temporaryTable, table);
+
+ checkFileStorePath(table, Arrays.asList("dt=2022-01-01", "dt=2022-01-02"));
+
+ // test batch read
+ testBatchRead(
+ buildSimpleQuery(table),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-01"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-01"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01")));
// overwrite dynamic partition
- prepareEnvAndOverwrite(
- managedTable,
- Collections.emptyMap(),
+ insertOverwrite(table, "('US Dollar', 'Thai Baht', 33.51, '2022-01-01')");
+
+ // batch read to check partition refresh
+ testBatchRead(
+ buildSimpleQuery(table),
Collections.singletonList(
- new String[] {"'US Dollar'", "'Thai Baht'", "33.51", "'2022-01-01'"}));
+ changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-01")));
- // streaming iter will not receive any changelog
- assertNoMoreRecords(streamIter);
+ // reset table
+ table =
+ createTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"));
- // batch read to check partition refresh
- collectAndCheck(
- tEnv,
- managedTable,
- Collections.emptyMap(),
- "dt = '2022-01-01'",
- Collections.singletonList(
- changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-01")))
- .close();
+ insertIntoFromTable(temporaryTable, table);
// test partition filter
- collectAndCheckBatchReadWrite(
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- "dt = '2022-01-02'",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE dt = '2022-01-02'"),
Arrays.asList(
changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02")));
// test field filter
- collectAndCheckBatchReadWrite(
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- "rate_by_to_currency < 0.1",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE rate_by_to_currency < 0.1"),
Arrays.asList(
changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02")));
// test partition and field filter
- collectAndCheckBatchReadWrite(
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- "rate_by_to_currency > 0.9 OR dt = '2022-01-02'",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE rate_by_to_currency > 0.9 OR dt = '2022-01-02'"),
Arrays.asList(
changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
@@ -339,17 +352,14 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02")));
// test projection
- collectAndCheckBatchReadWrite(
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- null,
- Arrays.asList("rate_by_to_currency", "dt"),
+ testBatchRead(
+ buildQuery(table, "rate_by_to_currency, dt", ""),
Arrays.asList(
changelogRow("+I", 1.0d, "2022-01-01"), // US Dollar,US Dollar
changelogRow("+I", 1.11d, "2022-01-01"), // Euro,US Dollar
changelogRow("+I", 0.13d, "2022-01-01"), // HK Dollar,US Dollar
changelogRow("+I", 0.0082d, "2022-01-01"), // Yen,US Dollar
- changelogRow("+I", 0.74d, "2022-01-01"), // Singapore Dollar,US Dollar
+ changelogRow("+I", 0.74d, "2022-01-01"), // Singapore Dollar,USDollar
changelogRow("+I", 0.9d, "2022-01-01"), // US Dollar,Euro
changelogRow("+I", 0.67d, "2022-01-01"), // Singapore Dollar,Euro
changelogRow("+I", 1.0d, "2022-01-01"), // Yen,Yen
@@ -360,11 +370,11 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
));
// test projection and filter
- collectAndCheckBatchReadWrite(
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- "dt = '2022-01-02' OR rate_by_to_currency > 100",
- Arrays.asList("from_currency", "to_currency"),
+ testBatchRead(
+ buildQuery(
+ table,
+ "from_currency, to_currency",
+ "WHERE dt = '2022-01-02' OR rate_by_to_currency > 100"),
Arrays.asList(
changelogRow("+I", "Yen", "US Dollar"),
changelogRow("+I", "US Dollar", "US Dollar"),
@@ -373,12 +383,13 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
@Test
public void testBatchWriteWithNonPartitionedRecordsWithMultiPk() throws Exception {
- // input is exchangeRates()
- List<Row> expectedRecords =
+ List<Row> initialRecords =
Arrays.asList(
// to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
changelogRow("+I", "Euro", "US Dollar", 1.11d),
changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d),
changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
changelogRow("+I", "Yen", "US Dollar", 0.0081d),
changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
@@ -388,63 +399,61 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
// to_currency is Yen
changelogRow("+I", "Yen", "Yen", 1.0d),
changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d),
changelogRow("+I", "Singapore Dollar", "Yen", 122.46d));
- // test batch read
- String managedTable =
- collectAndCheckBatchReadWrite(
- Collections.emptyList(), // partition
- Arrays.asList("from_currency", "to_currency"), // pk
- null,
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE"),
+ Arrays.asList("from_currency", "to_currency"),
Collections.emptyList(),
- expectedRecords);
-
- checkFileStorePath(tEnv, managedTable, null);
-
- // test streaming read
- final StreamTableEnvironment streamTableEnv =
- StreamTableEnvironment.create(buildStreamEnv());
- registerTable(streamTableEnv, managedTable);
- BlockingIterator<Row, Row> streamIter =
- collectAndCheck(
- streamTableEnv,
- managedTable,
- Collections.emptyMap(),
+ initialRecords,
null,
- expectedRecords);
+ true,
+ "I");
- // overwrite the whole table
- prepareEnvAndOverwrite(
- managedTable,
- Collections.emptyMap(),
- Collections.singletonList(new String[] {"'US Dollar'", "'Thai Baht'", "33.51"}));
+ String table =
+ createTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE"),
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.emptyList());
- // streaming iter will not receive any changelog
- assertNoMoreRecords(streamIter);
+ insertIntoFromTable(temporaryTable, table);
- // batch read to check data refresh
- collectAndCheck(
- tEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- Collections.singletonList(
- changelogRow("+I", "US Dollar", "Thai Baht", 33.51d)))
- .close();
+ checkFileStorePath(table, Collections.emptyList());
+
+ // test batch read
+ testBatchRead(
+ buildSimpleQuery(table),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "Euro", "US Dollar", 1.11d),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+ changelogRow("+I", "Singapore Dollar", "Yen", 122.46d)));
// test field filter
- collectAndCheckBatchReadWrite(
- Collections.emptyList(), // partition
- Arrays.asList("from_currency", "to_currency"), // pk
- "rate_by_to_currency < 0.1",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE rate_by_to_currency < 0.1"),
Collections.singletonList(changelogRow("+I", "Yen", "US Dollar", 0.0081d)));
// test projection
- collectAndCheckBatchReadWrite(
- Collections.emptyList(), // partition
- Arrays.asList("from_currency", "to_currency"), // pk
- null,
- Collections.singletonList("rate_by_to_currency"),
+ testBatchRead(
+ buildQuery(table, "rate_by_to_currency", ""),
Arrays.asList(
changelogRow("+I", 1.11d), // Euro, US Dollar
changelogRow("+I", 0.13d), // HK Dollar, US Dollar
@@ -459,86 +468,60 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
));
// test projection and filter
- collectAndCheckBatchReadWrite(
- Collections.emptyList(), // partition
- Arrays.asList("from_currency", "to_currency"), // pk
- "rate_by_to_currency > 100",
- Arrays.asList("from_currency", "to_currency"),
+ testBatchRead(
+ buildQuery(table, "from_currency, to_currency", "WHERE rate_by_to_currency > 100"),
Collections.singletonList(changelogRow("+I", "Singapore Dollar", "Yen")));
}
@Test
public void testBatchWriteMultiPartitionedRecordsWithOnePk() throws Exception {
- // input is hourlyRates()
- List<Row> expectedRecords =
+ List<Row> initialRecords =
Arrays.asList(
// dt = 2022-01-01, hh = 00
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "00"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
changelogRow("+I", "Yen", 1L, "2022-01-01", "00"),
changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
changelogRow("+I", "US Dollar", 114L, "2022-01-01", "00"),
// dt = 2022-01-01, hh = 20
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
// dt = 2022-01-02, hh = 12
changelogRow("+I", "Euro", 119L, "2022-01-02", "12"));
- // test batch read
- String managedTable =
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- null,
- Collections.emptyList(),
- expectedRecords);
-
- checkFileStorePath(tEnv, managedTable, hourlyRates().f1);
-
- // test streaming read
- final StreamTableEnvironment streamTableEnv =
- StreamTableEnvironment.create(buildStreamEnv());
- registerTable(streamTableEnv, managedTable);
- BlockingIterator<Row, Row> streamIter =
- collectAndCheck(
- streamTableEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- expectedRecords);
- // dynamic overwrite
- // INSERT OVERWRITE `manged_table-${uuid}` VALUES(...)
- prepareEnvAndOverwrite(
- managedTable,
- Collections.emptyMap(),
- Collections.singletonList(
- new String[] {"'HK Dollar'", "80", "'2022-01-01'", "'00'"}));
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Arrays.asList("currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ initialRecords,
+ "dt:2022-01-01,hh:00;dt:2022-01-01,hh:20;dt:2022-01-02,hh:12",
+ true,
+ "I");
- // INSERT OVERWRITE `manged_table-${uuid}` PARTITION (dt = '2022-01-02') VALUES(...)
- prepareEnvAndOverwrite(
- managedTable,
- Collections.singletonMap("dt", "'2022-01-02'"),
- Collections.singletonList(new String[] {"'Euro'", "120", "'12'"}));
+ String table =
+ createTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Arrays.asList("currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"));
- // batch read to check data refresh
- collectAndCheck(
- tEnv,
- managedTable,
- Collections.emptyMap(),
- "hh = '00' OR hh = '12'",
- Arrays.asList(
- changelogRow("+I", "HK Dollar", 80L, "2022-01-01", "00"),
- changelogRow("+I", "Euro", 120L, "2022-01-02", "12")));
+ insertIntoFromTable(temporaryTable, table);
- // streaming iter will not receive any changelog
- assertNoMoreRecords(streamIter);
+ checkFileStorePath(
+ table,
+ Arrays.asList("dt=2022-01-01,hh=00", "dt=2022-01-01,hh=20", "dt=2022-01-02,hh=12"));
- // test partition filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- "hh >= '10'",
- Collections.emptyList(),
+ // test batch read
+ testBatchRead(
+ buildSimpleQuery(table),
Arrays.asList(
+ // dt = 2022-01-01, hh = 00
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "00"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "00"),
// dt = 2022-01-01, hh = 20
changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
@@ -546,22 +529,23 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
// dt = 2022-01-02, hh = 12
changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
- // test field filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- "rate >= 119",
- Collections.emptyList(),
- Collections.singletonList(
- // dt = 2022-01-02, hh = 12
+ // test partition filter
+ testBatchRead(
+ buildQuery(table, "*", "WHERE hh >= '10'"),
+ Arrays.asList(
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+ // test field filter
+ testBatchRead(
+ buildQuery(table, "*", "WHERE rate >= 119"),
+ Collections.singletonList(changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+
// test projection
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- null,
- Collections.singletonList("hh"),
+ testBatchRead(
+ buildQuery(table, "hh", ""),
Arrays.asList(
changelogRow("+I", "00"), // Yen, 1L, 2022-01-01
changelogRow("+I", "00"), // Euro, 114L, 2022-01-01
@@ -573,93 +557,98 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
));
// test projection and filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- "rate > 100 AND hh >= '20'",
- Collections.singletonList("rate"),
- Collections.singletonList(changelogRow("+I", 114L)));
+ testBatchRead(
+ buildQuery(table, "rate", "WHERE rate > 100 AND hh >= '20'"),
+ Collections.nCopies(2, changelogRow("+I", 114L)));
}
@Test
public void testBatchWriteMultiPartitionedRecordsWithoutPk() throws Exception {
- // input is hourlyRates()
+ List<Row> initialRecords =
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 00
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "00"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "00"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "00"),
+ // dt = 2022-01-01, hh = 20
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
+ // dt = 2022-01-02, hh = 12
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "12"));
- // test batch read
- String managedTable =
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- null,
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
Collections.emptyList(),
- hourlyRates().f0);
-
- checkFileStorePath(tEnv, managedTable, hourlyRates().f1);
-
- // test streaming read
- final StreamTableEnvironment streamTableEnv =
- StreamTableEnvironment.create(buildStreamEnv());
- registerTable(streamTableEnv, managedTable);
- BlockingIterator<Row, Row> streamIter =
- collectAndCheck(
- streamTableEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- hourlyRates().f0);
+ Arrays.asList("dt", "hh"),
+ initialRecords,
+ "dt:2022-01-01,hh:00;dt:2022-01-01,hh:20;dt:2022-01-02,hh:12",
+ true,
+ "I");
+
+ String table =
+ createTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"));
+
+ insertIntoFromTable(temporaryTable, table);
+
+ checkFileStorePath(
+ table,
+ Arrays.asList("dt=2022-01-01,hh=00", "dt=2022-01-01,hh=20", "dt=2022-01-02,hh=12"));
+
+ // test batch read
+ testBatchRead(buildSimpleQuery(table), initialRecords);
// dynamic overwrite the whole table with null partitions
- String query =
- String.format(
- "INSERT OVERWRITE `%s` SELECT 'Yen', 1, CAST(null AS STRING), CAST(null AS STRING) FROM `%s`",
- managedTable, managedTable);
- prepareEnvAndOverwrite(managedTable, query);
+ bEnv.executeSql(
+ String.format(
+ "INSERT OVERWRITE `%s` SELECT 'Yen', 1, CAST(NULL AS STRING), CAST(NULL AS STRING) FROM `%s`",
+ table, table))
+ .await();
+
// check new partition path
- checkFileStorePath(tEnv, managedTable, "dt:null,hh:null");
+ checkFileStorePath(table, Collections.singletonList("dt=null,hh=null"));
// batch read to check data refresh
- collectAndCheck(
- tEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- Collections.singletonList(changelogRow("+I", "Yen", 1L, null, null)));
+ testBatchRead(
+ buildSimpleQuery(table),
+ Collections.nCopies(11, changelogRow("+I", "Yen", 1L, null, null)));
+
+ // reset table
+ table =
+ createTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"));
- // streaming iter will not receive any changelog
- assertNoMoreRecords(streamIter);
+ insertIntoFromTable(temporaryTable, table);
// test partition filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- "hh >= '10'",
- Collections.emptyList(),
+ testBatchRead(
+ buildQuery(table, "*", "WHERE hh >= '10'"),
Arrays.asList(
- // dt = 2022-01-01, hh = 20
changelogRow("+I", "US Dollar", 102L, "2022-01-01", "20"),
changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
- // dt = 2022-01-02, hh = 12
changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
// test field filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- "rate >= 119",
- Collections.emptyList(),
- Collections.singletonList(
- // dt = 2022-01-02, hh = 12
- changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+ testBatchRead(
+ buildQuery(table, "*", "WHERE rate >= 119"),
+ Collections.singletonList(changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
// test projection
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- null,
- Collections.singletonList("currency"),
+ testBatchRead(
+ buildQuery(table, "currency", ""),
Arrays.asList(
// dt = 2022-01-01, hh = 00
changelogRow("+I", "US Dollar"),
@@ -677,610 +666,8 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
changelogRow("+I", "Euro")));
// test projection and filter
- collectAndCheckBatchReadWrite(
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- "rate > 100 AND hh >= '20'",
- Collections.singletonList("rate"),
- Collections.singletonList(changelogRow("+I", 114L)));
- }
-
- @Test
- public void testEnableLogAndStreamingReadWriteSinglePartitionedRecordsWithMultiPk()
- throws Exception {
- // input is dailyExchangeRatesChangelogWithoutUB()
- // test hybrid read
- Tuple2<String, BlockingIterator<Row, Row>> tuple =
- collectAndCheckStreamingReadWriteWithoutClose(
- Collections.singletonList("dt"),
- Arrays.asList("from_currency", "to_currency", "dt"),
- Collections.emptyMap(),
- null,
- Collections.emptyList(),
- Arrays.asList(
- changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
- changelogRow(
- "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
- changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
- changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
- changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
- changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
- changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
- changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
- changelogRow(
- "+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-02")));
- String managedTable = tuple.f0;
- checkFileStorePath(tEnv, managedTable, dailyRatesChangelogWithoutUB().f1);
- BlockingIterator<Row, Row> streamIter = tuple.f1;
-
- // test streaming consume changelog
- tEnv.executeSql(
- String.format(
- "INSERT INTO `%s` PARTITION (dt = '2022-01-03')\n"
- + "VALUES('Chinese Yuan', 'HK Dollar', 1.231)\n",
- managedTable))
- .await();
-
- assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
- .containsExactlyInAnyOrderElementsOf(
- Collections.singletonList(
- changelogRow(
- "+I", "Chinese Yuan", "HK Dollar", 1.231d, "2022-01-03")));
-
- // dynamic overwrite the whole table
- String query =
- String.format(
- "INSERT OVERWRITE `%s` SELECT 'US Dollar', 'US Dollar', 1, '2022-04-02' FROM `%s`",
- managedTable, managedTable);
- prepareEnvAndOverwrite(managedTable, query);
- checkFileStorePath(tEnv, managedTable, "dt:2022-04-02");
-
- // batch read to check data refresh
- final StreamTableEnvironment batchTableEnv =
- StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
- registerTable(batchTableEnv, managedTable);
- collectAndCheck(
- batchTableEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- Collections.singletonList(
- changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-04-02")));
-
- // check no changelog generated for streaming read
- assertNoMoreRecords(streamIter);
-
- // filter on partition and field filter
- collectAndCheckStreamingReadWriteWithClose(
- true,
- Collections.singletonList("dt"),
- Arrays.asList("from_currency", "to_currency", "dt"),
- Collections.emptyMap(),
- "dt = '2022-01-02' AND from_currency = 'US Dollar'",
- Collections.emptyList(),
- Collections.singletonList(
- changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02")));
-
- // test projection and filter
- collectAndCheckStreamingReadWriteWithClose(
- true,
- Collections.singletonList("dt"),
- Arrays.asList("from_currency", "to_currency", "dt"),
- Collections.emptyMap(),
- "dt = '2022-01-01' AND rate_by_to_currency IS NULL",
- Arrays.asList("from_currency", "to_currency"),
- Collections.emptyList());
- }
-
- @Test
- public void testEnableLogAndStreamingReadWriteMultiPartitionedRecordsWithMultiPk()
- throws Exception {
- // input is hourlyExchangeRatesChangelogWithoutUB()
- // test hybrid read
- Tuple2<String, BlockingIterator<Row, Row>> tuple =
- collectAndCheckStreamingReadWriteWithoutClose(
- Arrays.asList("dt", "hh"),
- Arrays.asList("from_currency", "to_currency", "dt", "hh"),
- Collections.emptyMap(),
- null,
- Collections.emptyList(),
- Arrays.asList(
- changelogRow(
- "+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
- changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
- changelogRow(
- "+I",
- "Singapore Dollar",
- "US Dollar",
- 0.76d,
- "2022-01-02",
- "20"),
- changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
- changelogRow(
- "+I", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
- changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
- changelogRow(
- "+I", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
- changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
- changelogRow(
- "+I",
- "Singapore Dollar",
- "Yen",
- 90.1d,
- "2022-01-02",
- "20")));
- String managedTable = tuple.f0;
- checkFileStorePath(tEnv, managedTable, hourlyExchangeRatesChangelogWithoutUB().f1);
- BlockingIterator<Row, Row> streamIter = tuple.f1;
-
- // test streaming consume changelog
- tEnv.executeSql(
- String.format(
- "INSERT INTO `%s` \n"
- + "VALUES('Chinese Yuan', 'HK Dollar', 1.231, '2022-01-03', '15')\n",
- managedTable))
- .await();
-
- assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
- .containsExactlyInAnyOrderElementsOf(
- Collections.singletonList(
- changelogRow(
- "+I",
- "Chinese Yuan",
- "HK Dollar",
- 1.231d,
- "2022-01-03",
- "15")));
-
- // dynamic overwrite the whole table
- String query =
- String.format(
- "INSERT OVERWRITE `%s` SELECT 'US Dollar', 'US Dollar', 1, '2022-04-02', '10' FROM `%s`",
- managedTable, managedTable);
- prepareEnvAndOverwrite(managedTable, query);
- checkFileStorePath(tEnv, managedTable, "dt:2022-04-02,hh:10");
-
- // batch read to check data refresh
- final StreamTableEnvironment batchTableEnv =
- StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
- registerTable(batchTableEnv, managedTable);
- collectAndCheck(
- batchTableEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- Collections.singletonList(
- changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-04-02", "10")));
-
- // check no changelog generated for streaming read
- assertNoMoreRecords(streamIter);
-
- // filter on partition and field filter
- collectAndCheckStreamingReadWriteWithClose(
- true,
- Arrays.asList("dt", "hh"),
- Arrays.asList("from_currency", "to_currency", "dt", "hh"),
- Collections.emptyMap(),
- "dt = '2022-01-02' AND from_currency = 'US Dollar'",
- Collections.emptyList(),
- Arrays.asList(
- changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
- changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
-
- // test projection and filter
- collectAndCheckStreamingReadWriteWithClose(
- true,
- Arrays.asList("dt", "hh"),
- Arrays.asList("from_currency", "to_currency", "dt", "hh"),
- Collections.emptyMap(),
- "dt = '2022-01-02' AND from_currency = 'US Dollar'",
- Arrays.asList("from_currency", "to_currency"),
- Arrays.asList(
- changelogRow("+I", "US Dollar", "Euro"),
- changelogRow("+I", "US Dollar", "Yen")));
- }
-
- @Test
- public void testEnableLogAndStreamingReadWriteMultiPartitionedRecordsWithoutPk()
- throws Exception {
- // input is dailyExchangeRatesChangelogWithUB()
- // test hybrid read
- Tuple2<String, BlockingIterator<Row, Row>> tuple =
- collectAndCheckStreamingReadWriteWithoutClose(
- Collections.singletonList("dt"),
- Collections.emptyList(),
- Collections.emptyMap(),
- null,
- Collections.emptyList(),
- Arrays.asList(
- // dt = 2022-01-01
- changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
- // dt = 2022-01-02
- changelogRow("+I", "Euro", 115L, "2022-01-02")));
- String managedTable = tuple.f0;
- checkFileStorePath(tEnv, managedTable, dailyRatesChangelogWithUB().f1);
- BlockingIterator<Row, Row> streamIter = tuple.f1;
-
- // test streaming consume changelog
- tEnv.executeSql(
- String.format(
- "INSERT INTO `%s` PARTITION (dt = '2022-04-02')\n"
- + "VALUES('Euro', 116)\n",
- managedTable))
- .await();
-
- assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
- .containsExactlyInAnyOrderElementsOf(
- Collections.singletonList(changelogRow("+I", "Euro", 116L, "2022-04-02")));
-
- // dynamic overwrite the whole table
- String query =
- String.format(
- "INSERT OVERWRITE `%s` SELECT 'US Dollar', 103, '2022-04-02' FROM `%s`",
- managedTable, managedTable);
- prepareEnvAndOverwrite(managedTable, query);
- checkFileStorePath(tEnv, managedTable, "dt:2022-04-02");
-
- // batch read to check data refresh
- final StreamTableEnvironment batchTableEnv =
- StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
- registerTable(batchTableEnv, managedTable);
- collectAndCheck(
- batchTableEnv,
- managedTable,
- Collections.emptyMap(),
- null,
- Collections.singletonList(changelogRow("+I", "US Dollar", 103L, "2022-04-02")));
-
- // check no changelog generated for streaming read
- assertNoMoreRecords(streamIter);
-
- // filter on partition and field filter
- collectAndCheckStreamingReadWriteWithClose(
- true,
- Collections.singletonList("dt"),
- Collections.emptyList(),
- Collections.emptyMap(),
- "dt = '2022-01-01' AND currency = 'Yen'",
- Collections.emptyList(),
- Collections.emptyList());
-
- // test projection and filter
- collectAndCheckStreamingReadWriteWithClose(
- true,
- Collections.singletonList("dt"),
- Collections.emptyList(),
- Collections.emptyMap(),
- "dt = '2022-01-01' AND rate = 103",
- Collections.singletonList("currency"),
- Collections.emptyList());
- }
-
- @Test
- public void testReadLatestChangelogOfMultiPartitionedRecordsWithMultiPk() throws Exception {
- // input is hourlyExchangeRatesChangelogWithoutUB()
- List<Row> expectedRecords = new ArrayList<>(hourlyExchangeRatesChangelogWithoutUB().f0);
- expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"));
- expectedRecords.add(
- changelogRow("-U", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"));
- expectedRecords.add(
- changelogRow("-U", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"));
- expectedRecords.add(changelogRow("-U", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"));
- expectedRecords.add(
- changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"));
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- null,
- Collections.emptyList(),
- expectedRecords);
-
- // test partition filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "dt = '2022-01-02' AND hh = '21'",
- Collections.emptyList(),
- Collections.singletonList(
- changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
-
- // test field filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"),
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "rate_by_to_currency IS NOT NULL AND from_currency = 'US Dollar'",
- Collections.emptyList(),
- Arrays.asList(
- // to_currency is USD
- changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
- changelogRow("-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
- // to_currency is Euro
- changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
- // to_currency is Yen
- changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
-
- // test partition and field filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"),
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "hh = '21' AND from_currency = 'US Dollar'",
- Collections.emptyList(),
- Collections.singletonList(
- changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
-
- // test projection
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"),
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- null,
- Arrays.asList("from_currency", "to_currency"),
- Arrays.asList(
- // to_currency is USD
- changelogRow("+I", "US Dollar", "US Dollar"),
- changelogRow("+I", "Euro", "US Dollar"),
- changelogRow("+I", "HK Dollar", "US Dollar"),
- changelogRow("+I", "Yen", "US Dollar"),
- changelogRow("+I", "Singapore Dollar", "US Dollar"),
- changelogRow("-D", "US Dollar", "US Dollar"),
- changelogRow("-D", "Euro", "US Dollar"),
- // to_currency is Euro
- changelogRow("+I", "US Dollar", "Euro"),
- changelogRow("+I", "Singapore Dollar", "Euro"),
- // to_currency is Yen
- changelogRow("+I", "Yen", "Yen"),
- changelogRow("+I", "Chinese Yuan", "Yen"),
- changelogRow("+I", "Singapore Dollar", "Yen"),
- changelogRow("+I", "US Dollar", "Yen")));
-
- // test projection and filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"),
- Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
- "rate_by_to_currency > 100",
- Arrays.asList("from_currency", "to_currency"),
- Collections.singletonList(changelogRow("+I", "US Dollar", "Yen")));
- }
-
- @Test
- public void testReadLatestChangelogOfSinglePartitionedRecordsWithMultiPk() throws Exception {
- // input is dailyExchangeRatesChangelogWithoutUB()
- List<Row> expectedRecords = new ArrayList<>(dailyExchangeRatesChangelogWithoutUB().f0);
- expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"));
- expectedRecords.add(changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01"));
- expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"));
-
- collectLatestLogAndCheck(
- false,
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- null,
- Collections.emptyList(),
- expectedRecords);
-
- // test partition filter
- collectLatestLogAndCheck(
- false,
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- "dt = '2022-01-02'",
- Collections.emptyList(),
- Arrays.asList(
- changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
- changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
- changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
- changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
- changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
- changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
- changelogRow("+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"),
- changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02")));
-
- // test field filter
- collectLatestLogAndCheck(
- false,
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- "rate_by_to_currency IS NULL",
- Collections.emptyList(),
- Arrays.asList(
- changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
- changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
- changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"),
- changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01")));
-
- // test partition and field filter
- collectLatestLogAndCheck(
- false,
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- "dt = '2022-01-02' AND from_currency = 'Yen'",
- Collections.emptyList(),
- Collections.singletonList(changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02")));
-
- // test projection and filter
- collectLatestLogAndCheck(
- false,
- Collections.singletonList("dt"), // partition
- Arrays.asList("from_currency", "to_currency", "dt"), // pk
- "rate_by_to_currency > 100",
- Arrays.asList("from_currency", "to_currency"),
- Collections.singletonList(changelogRow("+U", "Singapore Dollar", "Yen")));
- }
-
- @Test
- public void testReadLatestChangelogOfNonPartitionedRecordsWithMultiPk() throws Exception {
- // input is exchangeRatesChangelogWithoutUB()
- List<Row> expectedRecords = new ArrayList<>(exchangeRatesChangelogWithoutUB());
- expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d));
- expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", 1.11d));
- expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Euro", 0.67d));
- expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d));
- expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 122.46d));
-
- collectLatestLogAndCheck(
- false,
- Collections.emptyList(), // partition
- Arrays.asList("from_currency", "to_currency"), // pk
- null,
- Collections.emptyList(),
- expectedRecords);
-
- // test field filter
- collectLatestLogAndCheck(
- false,
- Collections.emptyList(), // partition
- Arrays.asList("from_currency", "to_currency"), // pk
- "rate_by_to_currency < 1 OR rate_by_to_currency > 100",
- Collections.emptyList(),
- Arrays.asList(
- // to_currency is USD
- changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
- changelogRow("+I", "Yen", "US Dollar", 0.0082d),
- changelogRow("-U", "Yen", "US Dollar", 0.0082d),
- changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
- changelogRow("+U", "Yen", "US Dollar", 0.0081d),
- changelogRow("-D", "Yen", "US Dollar", 0.0081d),
- // to_currency is Euro
- changelogRow("+I", "US Dollar", "Euro", 0.9d),
- changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
- changelogRow("-U", "Singapore Dollar", "Euro", 0.67d),
- changelogRow("+U", "Singapore Dollar", "Euro", 0.69d),
- // to_currency is Yen
- changelogRow("+U", "Singapore Dollar", "Yen", 122.46d),
- changelogRow("-U", "Singapore Dollar", "Yen", 122.46d),
- changelogRow("+U", "Singapore Dollar", "Yen", 122d)));
-
- // test projection and filter
- collectLatestLogAndCheck(
- false,
- Collections.emptyList(), // partition
- Arrays.asList("from_currency", "to_currency"), // pk
- "rate_by_to_currency < 1 OR rate_by_to_currency > 100",
- Arrays.asList("from_currency", "to_currency"),
- Arrays.asList(
- // to_currency is USD
- changelogRow("+I", "HK Dollar", "US Dollar"),
- changelogRow("+I", "Yen", "US Dollar"),
- changelogRow("-U", "Yen", "US Dollar"),
- changelogRow("+I", "Singapore Dollar", "US Dollar"),
- changelogRow("+U", "Yen", "US Dollar"),
- changelogRow("-D", "Yen", "US Dollar"),
- // to_currency is Euro
- changelogRow("+I", "US Dollar", "Euro"),
- changelogRow("+I", "Singapore Dollar", "Euro"),
- changelogRow("-U", "Singapore Dollar", "Euro"),
- changelogRow("+U", "Singapore Dollar", "Euro"),
- // to_currency is Yen
- changelogRow("+U", "Singapore Dollar", "Yen"),
- changelogRow("-U", "Singapore Dollar", "Yen"),
- changelogRow("+U", "Singapore Dollar", "Yen")));
- }
-
- @Test
- public void testReadLatestChangelogOfMultiPartitionedRecordsWithOnePk() throws Exception {
- // input is hourlyRatesChangelogWithoutUB()
- List<Row> expectedRecords = new ArrayList<>(hourlyRatesChangelogWithoutUB().f0);
- expectedRecords.remove(changelogRow("+U", "Euro", 119L, "2022-01-02", "23"));
- expectedRecords.add(changelogRow("-U", "Euro", 114L, "2022-01-01", "15"));
-
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- null,
- Collections.emptyList(),
- expectedRecords);
-
- // test partition filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- "dt >= '2022-01-02'",
- Collections.emptyList(),
- Collections.singletonList(changelogRow("+I", "Euro", 119L, "2022-01-02", "23")));
-
- // test field filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- "rate = 1",
- Collections.emptyList(),
- Arrays.asList(
- changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
- changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
-
- // test projection and filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Arrays.asList("currency", "dt", "hh"), // pk
- "rate = 1",
- Collections.singletonList("currency"),
- Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")));
- }
-
- @Test
- public void testReadLatestChangelogOfMultiPartitionedRecordsWithoutPk() throws Exception {
- // input is hourlyRatesChangelogWithUB()
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- null,
- Collections.emptyList(),
- Arrays.asList(
- // dt = 2022-01-01, hh = 15
- changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
- changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
- changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
- changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
- changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
- // dt = 2022-01-02, hh = 20
- changelogRow("+I", "Euro", 114L, "2022-01-02", "20"),
- changelogRow("-D", "Euro", 114L, "2022-01-02", "20"),
- changelogRow("+I", "Euro", 119L, "2022-01-02", "20"),
- changelogRow("-D", "Euro", 119L, "2022-01-02", "20"),
- changelogRow("+I", "Euro", 115L, "2022-01-02", "20")));
-
- // test partition filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- "hh <> '20'",
- Collections.emptyList(),
- Arrays.asList(
- changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
- changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
- changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
- changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
- changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
-
- // test field filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- "rate = 1",
- Collections.emptyList(),
- Arrays.asList(
- changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
- changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
-
- // test projection and filter
- collectLatestLogAndCheck(
- false,
- Arrays.asList("dt", "hh"), // partition
- Collections.emptyList(), // pk
- "rate = 1",
- Collections.singletonList("currency"),
- Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")));
+ testBatchRead(
+ buildQuery(table, "rate", "WHERE rate > 110 AND hh >= '20'"),
+ Collections.nCopies(3, changelogRow("+I", 114L)));
}
}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableWIthKafkaLogITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableWIthKafkaLogITCase.java
new file mode 100644
index 00000000..0192e4ec
--- /dev/null
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableWIthKafkaLogITCase.java
@@ -0,0 +1,1134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.connector;
+
+import org.apache.flink.table.store.file.utils.BlockingIterator;
+import org.apache.flink.table.store.kafka.KafkaTableTestBase;
+import org.apache.flink.types.Row;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.SCAN_LATEST;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.assertNoMoreRecords;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.bEnv;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQuery;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQueryWithTableOptions;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildSimpleQuery;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.checkFileStorePath;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTableWithKafkaLog;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTemporaryTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.init;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertInto;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertIntoFromTable;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertIntoPartition;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testBatchRead;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingRead;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingReadWithReadFirst;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.validateStreamingReadResult;
+
+/**
+ * IT cases of streaming reading and writing tables which have composite primary keys and multiple
+ * partition fields with Kafka log.
+ */
+public class CompositePkAndMultiPartitionedTableWIthKafkaLogITCase extends KafkaTableTestBase {
+
+ @Before
+ public void setUp() throws Exception {
+ init(createAndRegisterTempFile("").toString());
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Non latest
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Test
+ public void testStreamingReadWriteMultiPartitionedRecordsWithMultiPk() throws Exception {
+ List<Row> initialRecords =
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", "US Dollar", null, "2022-01-02", "20"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"),
+ changelogRow("+U", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
+ changelogRow("-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", "US Dollar", null, "2022-01-02", "20"),
+ changelogRow(
+ "+U", "Singapore Dollar", "US Dollar", 0.76d, "2022-01-02", "20"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"),
+ changelogRow("+U", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"),
+ changelogRow("+U", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"),
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
+ changelogRow("+U", "Singapore Dollar", "Yen", 90.1d, "2022-01-02", "20"));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ initialRecords,
+ "dt:2022-01-02,hh:20;dt:2022-01-02,hh:21",
+ false,
+ "I,UA,D");
+
+ String table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ false);
+
+ insertIntoFromTable(temporaryTable, table);
+
+ checkFileStorePath(table, Arrays.asList("dt=2022-01-02,hh=20", "dt=2022-01-02,hh=21"));
+
+ BlockingIterator<Row, Row> streamingItr =
+ testStreamingRead(
+ buildSimpleQuery(table),
+ Arrays.asList(
+ changelogRow(
+ "+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
+ changelogRow(
+ "+I",
+ "Singapore Dollar",
+ "US Dollar",
+ 0.76d,
+ "2022-01-02",
+ "20"),
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ changelogRow(
+ "+I", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
+ changelogRow(
+ "+I", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
+ changelogRow(
+ "+I",
+ "Singapore Dollar",
+ "Yen",
+ 90.1d,
+ "2022-01-02",
+ "20")));
+
+ // test streaming consume changelog
+ insertInto(table, "('Chinese Yuan', 'HK Dollar', 1.231, '2022-01-03', '15')");
+
+ validateStreamingReadResult(
+ streamingItr,
+ Collections.singletonList(
+ changelogRow(
+ "+I", "Chinese Yuan", "HK Dollar", 1.231d, "2022-01-03", "15")));
+
+ // dynamic overwrite the whole table
+ bEnv.executeSql(
+ String.format(
+ "INSERT OVERWRITE `%s` SELECT 'US Dollar', 'US Dollar', 1, '2022-04-02', '10' FROM `%s`",
+ table, table))
+ .await();
+
+ checkFileStorePath(table, Collections.singletonList("dt=2022-04-02,hh=10"));
+
+ // batch read to check data refresh
+ testBatchRead(
+ buildSimpleQuery(table),
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-04-02", "10")));
+
+ // check no changelog generated for streaming read
+ assertNoMoreRecords(streamingItr);
+ streamingItr.close();
+
+ // reset table
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ false);
+
+ insertIntoFromTable(temporaryTable, table);
+
+ // filter on partition and field filter
+ testStreamingRead(
+ buildQuery(
+ table,
+ "*",
+ "WHERE dt = '2022-01-02' AND from_currency = 'US Dollar'"),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ changelogRow(
+ "+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")))
+ .close();
+
+ // test projection and filter
+ testStreamingRead(
+ buildQuery(
+ table,
+ "from_currency, to_currency",
+ "WHERE dt = '2022-01-02' AND from_currency = 'US Dollar'"),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "Euro"),
+ changelogRow("+I", "US Dollar", "Yen")))
+ .close();
+ }
+
+ @Test
+ public void testStreamingReadWriteSinglePartitionedRecordsWithMultiPk() throws Exception {
+ List<Row> initialRecords =
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("-D", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
+ changelogRow("+U", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+U", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"),
+ initialRecords,
+ "dt:2022-01-01;dt:2022-01-02",
+ false,
+ "I,UA,D");
+
+ String table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"),
+ false);
+
+ insertIntoFromTable(temporaryTable, table);
+
+ checkFileStorePath(table, Arrays.asList("dt=2022-01-01", "dt=2022-01-02"));
+
+ BlockingIterator<Row, Row> streamingItr =
+ testStreamingRead(
+ buildSimpleQuery(table),
+ Arrays.asList(
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+ changelogRow(
+ "+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-02")));
+
+ // test streaming consume changelog
+ insertIntoPartition(
+ table, "PARTITION (dt = '2022-01-03')", "('Chinese Yuan', 'HK Dollar', 1.231)");
+
+ validateStreamingReadResult(
+ streamingItr,
+ Collections.singletonList(
+ changelogRow("+I", "Chinese Yuan", "HK Dollar", 1.231d, "2022-01-03")));
+ streamingItr.close();
+
+ // filter on partition and field filter
+ testStreamingRead(
+ buildQuery(
+ table,
+ "*",
+ "WHERE dt = '2022-01-02' AND from_currency = 'US Dollar'"),
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02")))
+ .close();
+
+ // test projection and filter
+ testStreamingRead(
+ buildQuery(
+ table,
+ "from_currency, to_currency",
+ "WHERE dt = '2022-01-01' AND rate_by_to_currency IS NULL"),
+ Collections.emptyList())
+ .close();
+ }
+
+ @Test
+ public void testStreamingReadWriteMultiPartitionedRecordsWithoutPk() throws Exception {
+ List<Row> initialRecords =
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 10
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "10"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01", "10"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "10"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "10"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "10"),
+ // dt = 2022-01-02, hh = 10
+ changelogRow("+I", "Euro", 114L, "2022-01-02", "10"),
+ changelogRow("-U", "Euro", 114L, "2022-01-02", "10"),
+ changelogRow("+U", "Euro", 119L, "2022-01-02", "10"),
+ changelogRow("-D", "Euro", 119L, "2022-01-02", "10"),
+ changelogRow("+I", "Euro", 115L, "2022-01-02", "10"),
+ // dt = 2022-01-02, hh = 11
+ changelogRow("+I", "Yen", 1L, "2022-01-02", "11"));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"),
+ initialRecords,
+ "dt:2022-01-01,hh:10;dt:2022-01-02,hh:10;dt:2022-01-02,hh:11",
+ false,
+ "I,UA,UB,D");
+
+ String table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"),
+ false);
+
+ insertIntoFromTable(temporaryTable, table);
+
+ checkFileStorePath(
+ table,
+ Arrays.asList("dt=2022-01-01,hh=10", "dt=2022-01-02,hh=10", "dt=2022-01-02,hh=11"));
+
+ BlockingIterator<Row, Row> streamingItr =
+ testStreamingRead(
+ buildSimpleQuery(table),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "10"),
+ changelogRow("+I", "Euro", 115L, "2022-01-02", "10"),
+ changelogRow("+I", "Yen", 1L, "2022-01-02", "11")));
+
+ // test streaming consume changelog
+ insertIntoPartition(table, "PARTITION (dt = '2022-04-02')", "('Euro', 116, '10')");
+
+ validateStreamingReadResult(
+ streamingItr,
+ Collections.singletonList(changelogRow("+I", "Euro", 116L, "2022-04-02", "10")));
+ streamingItr.close();
+
+ // filter on partition and field filter
+ testStreamingRead(
+ buildQuery(table, "*", "WHERE dt = '2022-01-01' AND currency = 'Yen'"),
+ Collections.emptyList())
+ .close();
+
+ // test projection and filter
+ testStreamingRead(
+ buildQuery(table, "currency", "WHERE hh = '10' AND rate = 103"),
+ Collections.emptyList())
+ .close();
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Latest
+ // ----------------------------------------------------------------------------------------------------------------
+ @Test
+ public void testReadLatestChangelogOfMultiPartitionedRecordsWithMultiPk() throws Exception {
+ List<Row> initialRecords =
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", "US Dollar", null, "2022-01-02", "20"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"),
+ changelogRow("+U", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
+ changelogRow("-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", "US Dollar", null, "2022-01-02", "20"),
+ changelogRow(
+ "+U", "Singapore Dollar", "US Dollar", 0.76d, "2022-01-02", "20"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"),
+ changelogRow("+U", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"),
+ changelogRow("+U", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"),
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
+ changelogRow("+U", "Singapore Dollar", "Yen", 90.1d, "2022-01-02", "20"));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ initialRecords,
+ "dt:2022-01-02,hh:20;dt:2022-01-02,hh:21",
+ false,
+ "I,UA,D");
+
+ String table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ List<Row> expectedRecords = new ArrayList<>(initialRecords);
+ expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"));
+ expectedRecords.add(
+ changelogRow("-U", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"));
+ expectedRecords.add(
+ changelogRow("-U", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"));
+ expectedRecords.add(changelogRow("-U", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"));
+ expectedRecords.add(
+ changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"));
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+ expectedRecords)
+ .close();
+
+ // test partition filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table, "*", "WHERE dt = '2022-01-02' AND hh = '21'", SCAN_LATEST),
+ Collections.singletonList(
+ changelogRow(
+ "+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")))
+ .close();
+
+ // test field filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table,
+ "*",
+ "WHERE rate_by_to_currency IS NOT NULL AND from_currency = 'US Dollar'",
+ SCAN_LATEST),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow(
+ "+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ changelogRow(
+ "-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ // to_currency is Yen
+ changelogRow(
+ "+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")))
+ .close();
+
+ // test partition and field filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table,
+ "*",
+ "WHERE hh = '21' AND from_currency = 'US Dollar'",
+ SCAN_LATEST),
+ Collections.singletonList(
+ changelogRow(
+ "+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")))
+ .close();
+
+ // test projection
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table, "from_currency, to_currency", "", SCAN_LATEST),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar"),
+ changelogRow("+I", "Euro", "US Dollar"),
+ changelogRow("+I", "HK Dollar", "US Dollar"),
+ changelogRow("+I", "Yen", "US Dollar"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar"),
+ changelogRow("-D", "US Dollar", "US Dollar"),
+ changelogRow("-D", "Euro", "US Dollar"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro"),
+ changelogRow("+I", "Singapore Dollar", "Euro"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen"),
+ changelogRow("+I", "Chinese Yuan", "Yen"),
+ changelogRow("+I", "Singapore Dollar", "Yen"),
+ changelogRow("+I", "US Dollar", "Yen")))
+ .close();
+
+ // test projection and filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING",
+ "hh STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table,
+ "from_currency, to_currency",
+ "WHERE rate_by_to_currency > 100",
+ SCAN_LATEST),
+ Collections.singletonList(changelogRow("+I", "US Dollar", "Yen")))
+ .close();
+ }
+
+ @Test
+ public void testReadLatestChangelogOfSinglePartitionedRecordsWithMultiPk() throws Exception {
+ List<Row> initialRecords =
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("-D", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
+ changelogRow("+U", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+U", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"),
+ initialRecords,
+ "dt:2022-01-01;dt:2022-01-02",
+ false,
+ "I,UA,D");
+
+ String table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"),
+ true);
+
+ List<Row> expectedRecords = new ArrayList<>(initialRecords);
+ expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"));
+ expectedRecords.add(changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01"));
+ expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"));
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+ expectedRecords)
+ .close();
+
+ // test partition filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table, "*", "WHERE dt = '2022-01-02'", SCAN_LATEST),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
+ changelogRow(
+ "+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"),
+ changelogRow(
+ "-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02")))
+ .close();
+
+ // test field filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table, "*", "WHERE rate_by_to_currency IS NULL", SCAN_LATEST),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
+ changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"),
+ changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01")))
+ .close();
+
+ // test partition and field filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table,
+ "*",
+ "WHERE dt = '2022-01-02' AND from_currency = 'Yen'",
+ SCAN_LATEST),
+ Collections.singletonList(
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02")))
+ .close();
+
+ // test projection and filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE",
+ "dt STRING"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.singletonList("dt"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table,
+ "from_currency, to_currency",
+ "WHERE rate_by_to_currency > 100",
+ SCAN_LATEST),
+ Collections.singletonList(changelogRow("+U", "Singapore Dollar", "Yen")))
+ .close();
+ }
+
+ @Test
+ public void testReadLatestChangelogOfNonPartitionedRecordsWithMultiPk() throws Exception {
+ List<Row> initialRecords =
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+ changelogRow("+U", "Euro", "US Dollar", 1.12d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+ changelogRow("+U", "Yen", "US Dollar", 0.0081d),
+ changelogRow("-D", "US Dollar", "US Dollar", 1.0d),
+ changelogRow("-D", "Yen", "US Dollar", 0.0081d),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+ changelogRow("+U", "Singapore Dollar", "Euro", 0.69d),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d),
+ changelogRow("-D", "Yen", "Yen", 1.0d),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122.46d),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122d));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE"),
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.emptyList(),
+ initialRecords,
+ null,
+ false,
+ "I,UA,D");
+
+ String table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE"),
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.emptyList(),
+ true);
+
+ List<Row> expectedRecords = new ArrayList<>(initialRecords);
+ expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d));
+ expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", 1.11d));
+ expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Euro", 0.67d));
+ expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d));
+ expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 122.46d));
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+ expectedRecords)
+ .close();
+
+ // test field filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE"),
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.emptyList(),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table,
+ "*",
+ "WHERE rate_by_to_currency < 1 OR rate_by_to_currency > 100",
+ SCAN_LATEST),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d),
+ changelogRow("-U", "Yen", "US Dollar", 0.0082d),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+ changelogRow("+U", "Yen", "US Dollar", 0.0081d),
+ changelogRow("-D", "Yen", "US Dollar", 0.0081d),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+ changelogRow("-U", "Singapore Dollar", "Euro", 0.67d),
+ changelogRow("+U", "Singapore Dollar", "Euro", 0.69d),
+ // to_currency is Yen
+ changelogRow("+U", "Singapore Dollar", "Yen", 122.46d),
+ changelogRow("-U", "Singapore Dollar", "Yen", 122.46d),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122d)))
+ .close();
+
+ // test projection and filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList(
+ "from_currency STRING",
+ "to_currency STRING",
+ "rate_by_to_currency DOUBLE"),
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.emptyList(),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table,
+ "from_currency, to_currency",
+ "WHERE rate_by_to_currency < 1 OR rate_by_to_currency > 100",
+ SCAN_LATEST),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "HK Dollar", "US Dollar"),
+ changelogRow("+I", "Yen", "US Dollar"),
+ changelogRow("-U", "Yen", "US Dollar"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar"),
+ changelogRow("+U", "Yen", "US Dollar"),
+ changelogRow("-D", "Yen", "US Dollar"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro"),
+ changelogRow("+I", "Singapore Dollar", "Euro"),
+ changelogRow("-U", "Singapore Dollar", "Euro"),
+ changelogRow("+U", "Singapore Dollar", "Euro"),
+ // to_currency is Yen
+ changelogRow("+U", "Singapore Dollar", "Yen"),
+ changelogRow("-U", "Singapore Dollar", "Yen"),
+ changelogRow("+U", "Singapore Dollar", "Yen")))
+ .close();
+ }
+
+ @Test
+ public void testReadLatestChangelogOfMultiPartitionedRecordsWithOnePk() throws Exception {
+ List<Row> initialRecords =
+ Arrays.asList(
+ // dt = 2022-01-01, hh = "15"
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("+U", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ // dt = 2022-01-02, hh = "23"
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "23"),
+ changelogRow("+U", "Euro", 119L, "2022-01-02", "23"));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Arrays.asList("currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ initialRecords,
+ "dt:2022-01-01,hh:15;dt:2022-01-02,hh:23",
+ false,
+ "I,UA,D");
+
+ String table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Arrays.asList("currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ List<Row> expectedRecords = new ArrayList<>(initialRecords);
+ expectedRecords.remove(changelogRow("+U", "Euro", 119L, "2022-01-02", "23"));
+ expectedRecords.add(changelogRow("-U", "Euro", 114L, "2022-01-01", "15"));
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+ expectedRecords)
+ .close();
+
+ // test partition filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Arrays.asList("currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table, "*", "WHERE dt >= '2022-01-02'", SCAN_LATEST),
+ Collections.singletonList(
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "23")))
+ .close();
+
+ // test field filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Arrays.asList("currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "WHERE rate = 1", SCAN_LATEST),
+ Arrays.asList(
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15")))
+ .close();
+
+ // test projection and filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Arrays.asList("currency", "dt", "hh"),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table, "currency", "WHERE rate = 1", SCAN_LATEST),
+ Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")))
+ .close();
+ }
+
+ @Test
+ public void testReadLatestChangelogOfMultiPartitionedRecordsWithoutPk() throws Exception {
+ List<Row> initialRecords =
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 15
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+ // dt = 2022-01-02, hh = 20
+ changelogRow("+I", "Euro", 114L, "2022-01-02", "20"),
+ changelogRow("-U", "Euro", 114L, "2022-01-02", "20"),
+ changelogRow("+U", "Euro", 119L, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", 119L, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", 115L, "2022-01-02", "20"));
+
+ String temporaryTable =
+ createTemporaryTable(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"),
+ initialRecords,
+ "dt:2022-01-01,hh:15;dt:2022-01-02,hh:20",
+ false,
+ "I,UA,UB,D");
+
+ String table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 15
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+ // dt = 2022-01-02, hh = 20
+ changelogRow("+I", "Euro", 114L, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", 114L, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", 119L, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", 115L, "2022-01-02", "20")))
+ .close();
+
+ // test partition filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "WHERE hh <> '20'", SCAN_LATEST),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15")))
+ .close();
+
+ // test field filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "WHERE hh <> '20'", SCAN_LATEST),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15")))
+ .close();
+
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(table, "*", "WHERE rate = 1", SCAN_LATEST),
+ Arrays.asList(
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15")))
+ .close();
+
+ // test projection and filter
+ table =
+ createTableWithKafkaLog(
+ Arrays.asList("currency STRING", "rate BIGINT", "dt STRING", "hh STRING"),
+ Collections.emptyList(),
+ Arrays.asList("dt", "hh"),
+ true);
+
+ testStreamingReadWithReadFirst(
+ temporaryTable,
+ table,
+ buildQueryWithTableOptions(
+ table, "currency", "WHERE rate = 1", SCAN_LATEST),
+ Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")))
+ .close();
+ }
+}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
index 00d0a3f9..155898d3 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
@@ -24,22 +24,16 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.Schema;
import org.apache.flink.table.api.TableDescriptor;
-import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.table.catalog.CatalogBaseTable;
import org.apache.flink.table.catalog.ObjectIdentifier;
import org.apache.flink.table.catalog.ObjectPath;
import org.apache.flink.table.catalog.ResolvedCatalogTable;
-import org.apache.flink.table.store.CoreOptions.StartupMode;
import org.apache.flink.table.store.file.utils.BlockingIterator;
import org.apache.flink.table.store.kafka.KafkaTableTestBase;
import org.apache.flink.types.Row;
import javax.annotation.Nullable;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -47,18 +41,13 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-import static org.apache.flink.table.store.CoreOptions.SCAN_MODE;
import static org.apache.flink.table.store.connector.FlinkConnectorOptions.LOG_SYSTEM;
import static org.apache.flink.table.store.connector.FlinkConnectorOptions.ROOT_PATH;
-import static org.apache.flink.table.store.connector.FlinkConnectorOptions.relativeTablePath;
import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.prepareHelperSourceWithChangelogRecords;
import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.prepareHelperSourceWithInsertOnlyRecords;
import static org.apache.flink.table.store.connector.ShowCreateUtil.buildInsertIntoQuery;
-import static org.apache.flink.table.store.connector.ShowCreateUtil.buildInsertOverwriteQuery;
import static org.apache.flink.table.store.connector.ShowCreateUtil.buildSelectQuery;
-import static org.apache.flink.table.store.connector.ShowCreateUtil.buildSimpleSelectQuery;
import static org.apache.flink.table.store.connector.ShowCreateUtil.createTableLikeDDL;
import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
import static org.assertj.core.api.Assertions.assertThat;
@@ -69,71 +58,6 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
// ------------------------ Tools ----------------------------------
- protected void checkFileStorePath(
- StreamTableEnvironment tEnv, String managedTable, @Nullable String partitionList) {
- String relativeFilePath =
- relativeTablePath(
- ObjectIdentifier.of(
- tEnv.getCurrentCatalog(), tEnv.getCurrentDatabase(), managedTable));
- // check snapshot file path
- assertThat(Paths.get(rootPath, relativeFilePath, "snapshot")).exists();
- // check manifest file path
- assertThat(Paths.get(rootPath, relativeFilePath, "manifest")).exists();
- // check data file path
- if (partitionList == null) {
- // at least exists bucket-0
- assertThat(Paths.get(rootPath, relativeFilePath, "bucket-0")).exists();
- } else {
- Arrays.stream(partitionList.split(";"))
- .map(str -> str.replaceAll(":", "="))
- .map(str -> str.replaceAll(",", "/"))
- .map(str -> str.replaceAll("null", "__DEFAULT_PARTITION__"))
- .collect(Collectors.toList())
- .forEach(
- partition -> {
- assertThat(Paths.get(rootPath, relativeFilePath, partition))
- .exists();
- assertThat(
- Paths.get(
- rootPath,
- relativeFilePath,
- partition,
- "bucket-0"))
- .exists();
- });
- }
- }
-
- protected static BlockingIterator<Row, Row> collectAndCheck(
- StreamTableEnvironment tEnv,
- String managedTable,
- Map<String, String> hints,
- @Nullable String filter,
- List<Row> expectedRecords)
- throws Exception {
- List<Row> actual = new ArrayList<>();
- BlockingIterator<Row, Row> iterator =
- collect(
- tEnv,
- filter == null
- ? buildSimpleSelectQuery(managedTable, hints)
- : buildSelectQuery(
- managedTable, hints, filter, Collections.emptyList()),
- expectedRecords.size(),
- actual);
- assertThat(actual).containsExactlyInAnyOrderElementsOf(expectedRecords);
- return iterator;
- }
-
- protected static BlockingIterator<Row, Row> collect(
- StreamTableEnvironment tEnv, String selectQuery, int expectedSize, List<Row> actual)
- throws Exception {
- TableResult result = tEnv.executeSql(selectQuery);
- BlockingIterator<Row, Row> iterator = BlockingIterator.of(result.collect());
- actual.addAll(iterator.collect(expectedSize));
- return iterator;
- }
-
protected static void assertNoMoreRecords(BlockingIterator<Row, Row> iterator)
throws Exception {
List<Row> expectedRecords = Collections.emptyList();
@@ -148,107 +72,6 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
assertThat(expectedRecords).isEmpty();
}
- protected String collectAndCheckBatchReadWrite(
- List<String> partitions,
- List<String> primaryKeys,
- @Nullable String filter,
- List<String> projection,
- List<Row> expected)
- throws Exception {
- return collectAndCheckUnderSameEnv(
- false,
- false,
- true,
- partitions,
- primaryKeys,
- Collections.emptyList(),
- null,
- true,
- Collections.emptyMap(),
- filter,
- projection,
- expected)
- .f0;
- }
-
- protected String collectAndCheckStreamingReadWriteWithClose(
- boolean enableLogStore,
- List<String> partitions,
- List<String> primaryKeys,
- Map<String, String> readHints,
- @Nullable String filter,
- List<String> projection,
- List<Row> expected)
- throws Exception {
- Tuple2<String, BlockingIterator<Row, Row>> tuple =
- collectAndCheckUnderSameEnv(
- true,
- enableLogStore,
- false,
- partitions,
- primaryKeys,
- Collections.emptyList(),
- null,
- true,
- readHints,
- filter,
- projection,
- expected);
- tuple.f1.close();
- return tuple.f0;
- }
-
- protected Tuple2<String, BlockingIterator<Row, Row>>
- collectAndCheckStreamingReadWriteWithoutClose(
- List<String> partitions,
- List<String> primaryKeys,
- Map<String, String> readHints,
- @Nullable String filter,
- List<String> projection,
- List<Row> expected)
- throws Exception {
- return collectAndCheckUnderSameEnv(
- true,
- true,
- false,
- partitions,
- primaryKeys,
- Collections.emptyList(),
- null,
- true,
- readHints,
- filter,
- projection,
- expected);
- }
-
- protected void collectLatestLogAndCheck(
- boolean insertOnly,
- List<String> partitionKeys,
- List<String> primaryKeys,
- @Nullable String filter,
- List<String> projection,
- List<Row> expected)
- throws Exception {
- Map<String, String> hints = new HashMap<>();
- hints.put(SCAN_MODE.key(), StartupMode.LATEST.name().toLowerCase());
- collectAndCheckUnderSameEnv(
- true,
- true,
- insertOnly,
- partitionKeys,
- primaryKeys,
- Collections.emptyList(),
- null,
- false,
- hints,
- filter,
- projection,
- expected)
- .f1
- .close();
- }
-
protected Tuple2<String, String> createSourceAndManagedTable(
boolean streaming,
boolean enableLogStore,
@@ -366,32 +189,6 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
return Tuple2.of(managedTable, iterator);
}
- protected void prepareEnvAndOverwrite(
- String managedTable,
- Map<String, String> staticPartitions,
- List<String[]> overwriteRecords)
- throws Exception {
- prepareEnvAndOverwrite(
- managedTable,
- buildInsertOverwriteQuery(managedTable, staticPartitions, overwriteRecords));
- }
-
- protected void prepareEnvAndOverwrite(String managedTable, String query) throws Exception {
- final StreamTableEnvironment batchEnv =
- StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
- registerTable(batchEnv, managedTable);
- batchEnv.executeSql(query).await();
- }
-
- protected void registerTable(StreamTableEnvironment tEnvToRegister, String managedTable)
- throws Exception {
- String cat = tEnv.getCurrentCatalog();
- String db = tEnv.getCurrentDatabase();
- ObjectPath objectPath = new ObjectPath(db, managedTable);
- CatalogBaseTable table = tEnv.getCatalog(cat).get().getTable(objectPath);
- tEnvToRegister.getCatalog(cat).get().createTable(objectPath, table, false);
- }
-
protected static StreamExecutionEnvironment buildStreamEnv() {
return buildStreamEnv(2);
}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
index 1ddb1455..355bfa80 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
@@ -37,7 +37,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
-import java.util.stream.IntStream;
/**
* SHOW CREATE statement Util.
@@ -67,41 +66,6 @@ public class ShowCreateUtil {
return ddl.toString();
}
- public static String buildInsertOverwriteQuery(
- String managedTableName,
- Map<String, String> staticPartitions,
- List<String[]> overwriteRecords) {
- StringBuilder insertDmlBuilder =
- new StringBuilder(String.format("INSERT OVERWRITE `%s`", managedTableName));
- if (staticPartitions.size() > 0) {
- String partitionString =
- staticPartitions.entrySet().stream()
- .map(
- entry ->
- String.format(
- "%s = %s", entry.getKey(), entry.getValue()))
- .collect(Collectors.joining(", "));
- insertDmlBuilder.append(String.format("PARTITION (%s)", partitionString));
- }
- insertDmlBuilder.append("\n VALUES ");
- overwriteRecords.forEach(
- record -> {
- int arity = record.length;
- insertDmlBuilder.append("(");
- IntStream.range(0, arity)
- .forEach(i -> insertDmlBuilder.append(record[i]).append(", "));
-
- if (arity > 0) {
- int len = insertDmlBuilder.length();
- insertDmlBuilder.delete(len - 2, len);
- }
- insertDmlBuilder.append("), ");
- });
- int len = insertDmlBuilder.length();
- insertDmlBuilder.delete(len - 2, len);
- return insertDmlBuilder.toString();
- }
-
public static String buildInsertIntoQuery(String sourceTableName, String managedTableName) {
return buildInsertIntoQuery(
sourceTableName, managedTableName, Collections.emptyMap(), Collections.emptyMap());
@@ -132,10 +96,6 @@ public class ShowCreateUtil {
return insertDmlBuilder.toString();
}
- public static String buildSimpleSelectQuery(String tableName, Map<String, String> hints) {
- return buildSelectQuery(tableName, hints, null, Collections.emptyList());
- }
-
public static String buildSelectQuery(
String tableName,
Map<String, String> hints,
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
index 677c15eb..6327ed58 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/StreamingReadWriteTableWithKafkaLogITCase.java
@@ -19,13 +19,10 @@
package org.apache.flink.table.store.connector;
import org.apache.flink.table.store.CoreOptions;
-import org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil;
import org.apache.flink.table.store.file.utils.BlockingIterator;
import org.apache.flink.table.store.kafka.KafkaTableTestBase;
import org.apache.flink.types.Row;
-import org.apache.kafka.clients.admin.AdminClient;
-import org.apache.kafka.clients.admin.NewTopic;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@@ -35,19 +32,17 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.UUID;
import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
import static org.apache.flink.table.store.CoreOptions.SCAN_MODE;
import static org.apache.flink.table.store.CoreOptions.SCAN_TIMESTAMP_MILLIS;
-import static org.apache.flink.table.store.connector.FlinkConnectorOptions.LOG_SYSTEM;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.SCAN_LATEST;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.assertNoMoreRecords;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQuery;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildQueryWithTableOptions;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.buildSimpleQuery;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.checkFileStorePath;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTableWithKafkaLog;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.createTemporaryTable;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.init;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.insertInto;
@@ -57,20 +52,11 @@ import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testBatchRead;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingRead;
import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.testStreamingReadWithReadFirst;
-import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
-import static org.apache.flink.table.store.kafka.KafkaLogOptions.TOPIC;
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.apache.flink.table.store.connector.util.ReadWriteTableTestUtil.validateStreamingReadResult;
/** Streaming reading and writing with Kafka log IT cases. */
public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBase {
- private final Map<String, String> scanLatest =
- new HashMap<String, String>() {
- {
- put(SCAN_MODE.key(), CoreOptions.StartupMode.LATEST.toString());
- }
- };
-
@Before
public void setUp() throws Exception {
init(createAndRegisterTempFile("").toString());
@@ -107,10 +93,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
- Collections.singletonList("dt"));
+ Collections.singletonList("dt"),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -132,11 +119,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
insertIntoPartition(table, "PARTITION (dt = '2022-01-04')", "('Yen', 20)");
- assertThat(streamItr.collect(2))
- .containsExactlyInAnyOrderElementsOf(
- Arrays.asList(
- changelogRow("+I", "HK Dollar", 100L, "2022-01-03"),
- changelogRow("+I", "Yen", 20L, "2022-01-03")));
+ validateStreamingReadResult(
+ streamItr,
+ Arrays.asList(
+ changelogRow("+I", "HK Dollar", 100L, "2022-01-03"),
+ changelogRow("+I", "Yen", 20L, "2022-01-03")));
// overwrite partition 2022-01-02
insertOverwritePartition(
@@ -216,10 +203,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,UB,D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Collections.emptyList(),
- Collections.singletonList("dt"));
+ Collections.singletonList("dt"),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -298,10 +286,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I, UA, D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
- Collections.emptyList());
+ Collections.emptyList(),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -360,10 +349,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,UB,D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
- Collections.emptyList());
+ Collections.emptyList(),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -433,7 +423,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
Collections.singletonList("dt"),
@@ -443,7 +433,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "*", "", scanLatest),
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
changelogRow("+I", "Euro", 114L, "2022-01-01"),
@@ -456,20 +446,20 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test only read the latest log
insertInto(table, "('US Dollar', 104, '2022-01-01')", "('Euro', 100, '2022-01-02')");
- assertThat(streamItr.collect(4))
- .containsExactlyInAnyOrderElementsOf(
- Arrays.asList(
- changelogRow("-U", "US Dollar", 102L, "2022-01-01"),
- changelogRow("+U", "US Dollar", 104L, "2022-01-01"),
- changelogRow("-U", "Euro", 119L, "2022-01-02"),
- changelogRow("+U", "Euro", 100L, "2022-01-02")));
+ validateStreamingReadResult(
+ streamItr,
+ Arrays.asList(
+ changelogRow("-U", "US Dollar", 102L, "2022-01-01"),
+ changelogRow("+U", "US Dollar", 104L, "2022-01-01"),
+ changelogRow("-U", "Euro", 119L, "2022-01-02"),
+ changelogRow("+U", "Euro", 100L, "2022-01-02")));
assertNoMoreRecords(streamItr);
streamItr.close();
// test partition filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
Collections.singletonList("dt"),
@@ -479,7 +469,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "*", "WHERE dt = '2022-01-01'", scanLatest),
+ table, "*", "WHERE dt = '2022-01-01'", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
changelogRow("+I", "Euro", 114L, "2022-01-01"),
@@ -492,7 +482,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test field filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
Collections.singletonList("dt"),
@@ -502,14 +492,14 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "*", "WHERE currency = 'Yen'", scanLatest),
+ table, "*", "WHERE currency = 'Yen'", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "Yen", 1L, "2022-01-01"),
changelogRow("-D", "Yen", 1L, "2022-01-01")))
.close();
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
Collections.singletonList("dt"),
@@ -518,7 +508,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "*", "WHERE rate = 114", scanLatest),
+ buildQueryWithTableOptions(table, "*", "WHERE rate = 114", SCAN_LATEST),
Arrays.asList(
// part = 2022-01-01
changelogRow("+I", "Euro", 114L, "2022-01-01"),
@@ -527,7 +517,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test partition and field filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
Collections.singletonList("dt"),
@@ -537,13 +527,13 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "*", "WHERE rate = 114 AND dt = '2022-01-02'", scanLatest),
+ table, "*", "WHERE rate = 114 AND dt = '2022-01-02'", SCAN_LATEST),
Collections.emptyList())
.close();
// test projection
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
Collections.singletonList("dt"),
@@ -552,7 +542,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "rate", "", scanLatest),
+ buildQueryWithTableOptions(table, "rate", "", SCAN_LATEST),
Arrays.asList(
// part = 2022-01-01
changelogRow("+I", 102L), // US Dollar
@@ -569,7 +559,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test projection and filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
Collections.singletonList("dt"),
@@ -579,7 +569,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "rate", "WHERE dt = '2022-01-02'", scanLatest),
+ table, "rate", "WHERE dt = '2022-01-02'", SCAN_LATEST),
Collections.singletonList(changelogRow("+I", 119L)) // Euro
)
.close();
@@ -613,7 +603,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,UB,D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Collections.emptyList(),
Collections.singletonList("dt"),
@@ -622,7 +612,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "*", "", scanLatest),
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
Arrays.asList(
// part = 2022-01-01
changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
@@ -640,7 +630,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test partition filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Collections.emptyList(),
Collections.singletonList("dt"),
@@ -650,7 +640,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "*", "WHERE dt = '2022-01-01'", scanLatest),
+ table, "*", "WHERE dt = '2022-01-01'", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
changelogRow("+I", "Euro", 116L, "2022-01-01"),
@@ -664,7 +654,10 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "*", "WHERE currency = 'US Dollar' OR rate = 1", scanLatest),
+ table,
+ "*",
+ "WHERE currency = 'US Dollar' OR rate = 1",
+ SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
changelogRow("+I", "Yen", 1L, "2022-01-01"),
@@ -673,7 +666,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test partition and field filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Collections.emptyList(),
Collections.singletonList("dt"),
@@ -683,7 +676,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "*", "WHERE dt = '2022-01-02' AND rate = 114", scanLatest),
+ table, "*", "WHERE dt = '2022-01-02' AND rate = 114", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "Euro", 114L, "2022-01-02"),
changelogRow("-D", "Euro", 114L, "2022-01-02")))
@@ -691,7 +684,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test projection
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Collections.emptyList(),
Collections.singletonList("dt"),
@@ -700,7 +693,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "rate", "", scanLatest),
+ buildQueryWithTableOptions(table, "rate", "", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", 102L),
changelogRow("+I", 116L),
@@ -716,7 +709,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test projection and filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Collections.emptyList(),
Collections.singletonList("dt"),
@@ -726,7 +719,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "rate", "WHERE dt <> '2022-01-01'", scanLatest),
+ table, "rate", "WHERE dt <> '2022-01-01'", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", 114L),
changelogRow("-D", 114L),
@@ -760,7 +753,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
Collections.emptyList(),
@@ -769,7 +762,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "*", "", scanLatest),
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L),
changelogRow("+I", "Euro", 114L),
@@ -783,7 +776,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test field filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
Collections.emptyList(),
@@ -793,7 +786,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "*", "WHERE currency = 'Euro'", scanLatest),
+ table, "*", "WHERE currency = 'Euro'", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "Euro", 114L),
changelogRow("-U", "Euro", 114L),
@@ -804,7 +797,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test projection
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
Collections.emptyList(),
@@ -813,7 +806,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "currency", "", scanLatest),
+ buildQueryWithTableOptions(table, "currency", "", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar"),
changelogRow("+I", "Euro"),
@@ -825,7 +818,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test projection and filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
Collections.emptyList(),
@@ -835,7 +828,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "rate", "WHERE currency = 'Euro'", scanLatest),
+ table, "rate", "WHERE currency = 'Euro'", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", 114L),
changelogRow("-U", 114L),
@@ -873,7 +866,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,UB,D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
Collections.emptyList(),
@@ -882,7 +875,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "*", "", scanLatest),
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L),
changelogRow("+I", "Euro", 114L),
@@ -900,7 +893,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test field filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
Collections.emptyList(),
@@ -910,7 +903,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "*", "WHERE currency = 'Euro'", scanLatest),
+ table, "*", "WHERE currency = 'Euro'", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "Euro", 114L),
changelogRow("-D", "Euro", 114L),
@@ -923,7 +916,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test projection
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
Collections.emptyList(),
@@ -932,7 +925,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "currency, rate", "", scanLatest),
+ buildQueryWithTableOptions(table, "currency, rate", "", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L),
changelogRow("+I", "Euro", 114L),
@@ -950,7 +943,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test projection and filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
Collections.emptyList(),
@@ -960,7 +953,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "currency", "WHERE currency IS NOT NULL", scanLatest),
+ table, "currency", "WHERE currency IS NOT NULL", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar"),
changelogRow("+I", "Euro"),
@@ -976,7 +969,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
.close();
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
Collections.emptyList(),
@@ -986,7 +979,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "currency", "WHERE rate = 119", scanLatest),
+ table, "currency", "WHERE rate = 119", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "Euro"),
changelogRow("-D", "Euro"),
@@ -1016,7 +1009,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
Collections.emptyList(),
@@ -1025,7 +1018,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "*", "", scanLatest),
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
initialRecords)
.close();
@@ -1041,7 +1034,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I");
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
Collections.emptyList(),
@@ -1050,7 +1043,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "*", "", scanLatest),
+ buildQueryWithTableOptions(table, "*", "", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L),
changelogRow("+I", "Euro", 114L),
@@ -1061,7 +1054,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test field filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
Collections.emptyList(),
@@ -1070,14 +1063,14 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "*", "WHERE rate = 114", scanLatest),
+ buildQueryWithTableOptions(table, "*", "WHERE rate = 114", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", "Euro", 114L), changelogRow("-U", "Euro", 114L)))
.close();
// test projection
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
Collections.emptyList(),
@@ -1086,7 +1079,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
testStreamingReadWithReadFirst(
temporaryTable,
table,
- buildQueryWithTableOptions(table, "rate", "", scanLatest),
+ buildQueryWithTableOptions(table, "rate", "", SCAN_LATEST),
Arrays.asList(
changelogRow("+I", 102L),
changelogRow("+I", 114L),
@@ -1097,7 +1090,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// test projection and filter
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
Collections.emptyList(),
@@ -1107,7 +1100,7 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
temporaryTable,
table,
buildQueryWithTableOptions(
- table, "currency", "WHERE rate = 114", scanLatest),
+ table, "currency", "WHERE rate = 114", SCAN_LATEST),
Arrays.asList(changelogRow("+I", "Euro"), changelogRow("-U", "Euro")))
.close();
}
@@ -1142,10 +1135,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Collections.emptyList(),
- Collections.singletonList("dt"));
+ Collections.singletonList("dt"),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -1166,10 +1160,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I");
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
- Collections.singletonList("dt"));
+ Collections.singletonList("dt"),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -1205,10 +1200,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I");
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.singletonList("currency"),
- Collections.emptyList());
+ Collections.emptyList(),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -1234,10 +1230,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I");
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
- Collections.emptyList());
+ Collections.emptyList(),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -1277,10 +1274,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT"),
Collections.emptyList(),
- Collections.emptyList());
+ Collections.emptyList(),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -1321,10 +1319,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,UB,D");
String table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Collections.emptyList(),
- Collections.singletonList("dt"));
+ Collections.singletonList("dt"),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -1370,10 +1369,11 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
"I,UA,D");
table =
- createTable(
+ createTableWithKafkaLog(
Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"),
Arrays.asList("currency", "dt"),
- Collections.singletonList("dt"));
+ Collections.singletonList("dt"),
+ false);
insertIntoFromTable(temporaryTable, table);
@@ -1395,50 +1395,6 @@ public class StreamingReadWriteTableWithKafkaLogITCase extends KafkaTableTestBas
// Tools
// ----------------------------------------------------------------------------------------------------------------
- private String createTable(
- List<String> fieldsSpec, List<String> primaryKeys, List<String> partitionKeys)
- throws Exception {
- return createTable(fieldsSpec, primaryKeys, partitionKeys, false);
- }
-
- private String createTable(
- List<String> fieldsSpec,
- List<String> primaryKeys,
- List<String> partitionKeys,
- boolean manuallyCreateLogTable)
- throws Exception {
- String topic = "topic_" + UUID.randomUUID();
- String table =
- ReadWriteTableTestUtil.createTable(
- fieldsSpec,
- primaryKeys,
- partitionKeys,
- new HashMap<String, String>() {
- {
- put(LOG_SYSTEM.key(), "kafka");
- put(BOOTSTRAP_SERVERS.key(), getBootstrapServers());
- put(TOPIC.key(), topic);
- }
- });
-
- if (manuallyCreateLogTable) {
- createKafkaLogTable(topic);
- }
-
- return table;
- }
-
- private void createKafkaLogTable(String topic) throws Exception {
- Properties properties = new Properties();
- properties.put("bootstrap.servers", getBootstrapServers());
-
- try (AdminClient adminClient = AdminClient.create(properties)) {
- NewTopic topicObj =
- new NewTopic(topic, Optional.of(1), Optional.empty()).configs(new HashMap<>());
- adminClient.createTopics(Collections.singleton(topicObj)).all().get();
- }
- }
-
private Map<String, String> scanFromTimeStampMillis(Long timeStampMillis) {
return new HashMap<String, String>() {
{
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
index 22279f28..b5b5f592 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/action/CompactActionITCase.java
@@ -32,11 +32,13 @@ import org.apache.flink.table.store.table.source.snapshot.SnapshotEnumerator;
import org.apache.flink.table.store.types.DataType;
import org.apache.flink.table.store.types.DataTypes;
import org.apache.flink.table.store.types.RowType;
+import org.apache.flink.table.store.utils.CommonTestUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -178,9 +180,13 @@ public class CompactActionITCase extends ActionITCaseBase {
60_000);
// assert dedicated compact job will expire snapshots
- Assertions.assertEquals(
- snapshotManager.latestSnapshotId() - 2,
- (long) snapshotManager.earliestSnapshotId());
+ CommonTestUtils.waitUtil(
+ () ->
+ snapshotManager.latestSnapshotId() - 2
+ == snapshotManager.earliestSnapshotId(),
+ Duration.ofSeconds(60_000),
+ Duration.ofSeconds(100),
+ String.format("Cannot validate snapshot expiration in %s milliseconds.", 60_000));
}
private List<Map<String, String>> getSpecifiedPartitions() {
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
index f16c90af..72890569 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/util/ReadWriteTableTestUtil.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.table.store.CoreOptions;
import org.apache.flink.table.store.connector.ReadWriteTableITCase;
import org.apache.flink.table.store.connector.StreamingReadWriteTableWithKafkaLogITCase;
import org.apache.flink.table.store.file.utils.BlockingIterator;
@@ -44,6 +45,12 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.apache.flink.table.planner.factories.TestValuesTableFactory.registerData;
+import static org.apache.flink.table.store.CoreOptions.SCAN_MODE;
+import static org.apache.flink.table.store.connector.FlinkConnectorOptions.LOG_SYSTEM;
+import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
+import static org.apache.flink.table.store.kafka.KafkaLogOptions.TOPIC;
+import static org.apache.flink.table.store.kafka.KafkaTableTestBase.createTopicIfNotExists;
+import static org.apache.flink.table.store.kafka.KafkaTableTestBase.getBootstrapServers;
import static org.assertj.core.api.Assertions.assertThat;
/**
@@ -55,6 +62,13 @@ public class ReadWriteTableTestUtil {
public static final int DEFAULT_PARALLELISM = 2;
+ public static final Map<String, String> SCAN_LATEST =
+ new HashMap<String, String>() {
+ {
+ put(SCAN_MODE.key(), CoreOptions.StartupMode.LATEST.toString());
+ }
+ };
+
public static TableEnvironment sEnv;
public static StreamExecutionEnvironment bExeEnv;
@@ -113,6 +127,32 @@ public class ReadWriteTableTestUtil {
return table;
}
+ public static String createTableWithKafkaLog(
+ List<String> fieldsSpec,
+ List<String> primaryKeys,
+ List<String> partitionKeys,
+ boolean manuallyCreateLogTable) {
+ String topic = "topic_" + UUID.randomUUID();
+ String table =
+ createTable(
+ fieldsSpec,
+ primaryKeys,
+ partitionKeys,
+ new HashMap<String, String>() {
+ {
+ put(LOG_SYSTEM.key(), "kafka");
+ put(BOOTSTRAP_SERVERS.key(), getBootstrapServers());
+ put(TOPIC.key(), topic);
+ }
+ });
+
+ if (manuallyCreateLogTable) {
+ createTopicIfNotExists(topic, 1);
+ }
+
+ return table;
+ }
+
public static String createTemporaryTable(
List<String> fieldsSpec,
List<String> primaryKeys,
@@ -208,6 +248,7 @@ public class ReadWriteTableTestUtil {
}
partitionSpec.stream()
.map(str -> str.replaceAll(",", "/"))
+ .map(str -> str.replaceAll("null", "__DEFAULT_PARTITION__"))
.forEach(
partition -> {
assertThat(Paths.get(warehouse, relativeFilePath, partition)).exists();
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/kafka/KafkaTableTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/kafka/KafkaTableTestBase.java
index aff25de2..5a0e3aa1 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/kafka/KafkaTableTestBase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/kafka/KafkaTableTestBase.java
@@ -122,7 +122,7 @@ public abstract class KafkaTableTestBase extends AbstractTestBase {
deleteTopics();
}
- public Properties getStandardProps() {
+ public static Properties getStandardProps() {
Properties standardProps = new Properties();
standardProps.put("bootstrap.servers", KAFKA_CONTAINER.getBootstrapServers());
standardProps.put("group.id", "flink-tests");
@@ -134,7 +134,7 @@ public abstract class KafkaTableTestBase extends AbstractTestBase {
return standardProps;
}
- public String getBootstrapServers() {
+ public static String getBootstrapServers() {
return KAFKA_CONTAINER.getBootstrapServers();
}
@@ -142,7 +142,7 @@ public abstract class KafkaTableTestBase extends AbstractTestBase {
return describeExternalTopics().containsKey(topicName);
}
- protected void createTopicIfNotExists(String topicName, int numBucket) {
+ public static void createTopicIfNotExists(String topicName, int numBucket) {
try (final AdminClient adminClient = AdminClient.create(getStandardProps())) {
if (!adminClient.listTopics().names().get().contains(topicName)) {
adminClient