You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2022/04/14 07:14:13 UTC
[flink-table-store] branch release-0.1 updated (cf58bff -> 6c9f3d3)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
from cf58bff [hotfix] Remove java reflect in TableConfigUtils
new 69ecd0d [FLINK-27070] Reuse FileFormat instead of DecodingFormat/EncodingFormat to ensure thread safety
new d466ec7 [FLINK-27085] Introduce snapshot.num-retained.min option
new 71676d2 [FLINK-26856] Add ITCase for compound primary key and multi-partition
new feb11f1 [FLINK-26913] Add IT case for computed column and watermark
new e5d9202 [hotfix] Add random group id to kafka consumer
new 77192d7 [hotfix] Value field should be named in value count mode
new f8c1351 [hotfix] Fix junit4 dependency version
new 7cbef7f [hotfix] Shade kafka dependencies
new 2385f68 [FLINK-27135] Rename 'file.path' table option in table store to 'path'
new d123325 [FLINK-27136] Disable rolling file when writing level 0 sst files
new 1c25a8d [FLINK-27158] Adjust default value of compaction and compress
new 2a527a6 [FLINK-27139] FileStoreCommitImpl#filterCommitted should exit when faced with expired files
new 88fcc83 [FLINK-27157] Add version field to manifest entries
new d27c934 [hotfix] Add logs to file store compact for better tunning
new 6c9f3d3 [FLINK-27123] Introduce filter predicate for 'LIKE' and fix filter pushdown bug
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/content/docs/development/create-table.md | 4 +-
docs/content/docs/development/write-table.md | 9 +-
docs/content/docs/try-table-store/quick-start.md | 2 +-
.../flink/table/store/connector/TableStore.java | 5 +-
.../store/connector/source/TableStoreSource.java | 56 +-
.../CompositePkAndMultiPartitionedTableITCase.java | 1288 ++++++++++++++++++++
.../ComputedColumnAndWatermarkTableITCase.java | 261 ++++
.../store/connector/ContinuousFileStoreITCase.java | 4 +-
.../table/store/connector/CreateTableITCase.java | 2 +-
.../table/store/connector/DropTableITCase.java | 2 +-
.../table/store/connector/FileStoreITCase.java | 6 +-
.../store/connector/ReadWriteTableITCase.java | 937 +++++---------
.../store/connector/ReadWriteTableTestBase.java | 452 +++++++
.../store/connector/ReadWriteTableTestUtil.java | 580 +++++++++
.../table/store/connector/ShowCreateUtil.java | 162 ++-
.../store/connector/TableStoreFactoryTest.java | 20 +-
.../table/store/connector/TableStoreTestBase.java | 41 +-
.../flink/table/store/file/FileStoreImpl.java | 3 +-
.../flink/table/store/file/FileStoreOptions.java | 41 +-
.../flink/table/store/file/format/FileFormat.java | 11 +-
.../table/store/file/format/FileFormatFactory.java | 4 +-
.../table/store/file/format/FileFormatImpl.java | 20 +-
.../file/manifest/ManifestEntrySerializer.java | 16 +-
.../table/store/file/manifest/ManifestFile.java | 15 +-
.../file/manifest/ManifestFileMetaSerializer.java | 16 +-
.../table/store/file/manifest/ManifestList.java | 14 +-
.../store/file/mergetree/MergeTreeOptions.java | 8 +-
.../file/mergetree/compact/CompactManager.java | 75 +-
.../mergetree/compact/UniversalCompaction.java | 14 +
.../store/file/mergetree/sst/SstFileReader.java | 11 +-
.../store/file/mergetree/sst/SstFileWriter.java | 17 +-
.../store/file/operation/FileStoreCommitImpl.java | 14 +
.../store/file/operation/FileStoreExpireImpl.java | 17 +-
.../store/file/operation/FileStoreWriteImpl.java | 8 +-
.../store/file/predicate/PredicateConverter.java | 100 +-
.../predicate/{LessThan.java => StartsWith.java} | 42 +-
.../file/utils/VersionedObjectSerializer.java | 66 +
.../flink/table/store/file/TestFileStore.java | 10 +-
.../FileStatsExtractingAvroFormatFactory.java | 4 +-
.../store/file/operation/FileStoreCommitTest.java | 17 +
.../store/file/operation/FileStoreExpireTest.java | 27 +-
.../table/store/file/predicate/PredicateTest.java | 495 +++++++-
flink-table-store-dist/pom.xml | 8 +
.../table/store/format/avro/AvroFileFormat.java | 8 +-
.../store/format/avro/AvroFileFormatFactory.java | 4 +-
.../table/store/format/orc/OrcFileFormat.java | 20 +-
.../store/format/orc/OrcFileFormatFactory.java | 21 +-
.../table/store/format/orc/OrcFileFormatTest.java | 48 +
.../table/store/kafka/KafkaLogSourceProvider.java | 2 +
pom.xml | 2 +-
50 files changed, 4183 insertions(+), 826 deletions(-)
create mode 100644 flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
create mode 100644 flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ComputedColumnAndWatermarkTableITCase.java
create mode 100644 flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
create mode 100644 flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestUtil.java
copy flink-table-store-core/src/main/java/org/apache/flink/table/store/file/predicate/{LessThan.java => StartsWith.java} (59%)
create mode 100644 flink-table-store-core/src/main/java/org/apache/flink/table/store/file/utils/VersionedObjectSerializer.java
create mode 100644 flink-table-store-format/src/test/java/org/apache/flink/table/store/format/orc/OrcFileFormatTest.java
[flink-table-store] 03/15: [FLINK-26856] Add ITCase for compound primary key and multi-partition
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 71676d2697b51c0c12ce0a86b67c7507417da7c5
Author: Jane Chan <55...@users.noreply.github.com>
AuthorDate: Thu Apr 7 09:20:25 2022 +0800
[FLINK-26856] Add ITCase for compound primary key and multi-partition
This closes #73
---
.../CompositePkAndMultiPartitionedTableITCase.java | 1286 ++++++++++++++++++++
.../table/store/connector/CreateTableITCase.java | 2 +-
.../table/store/connector/DropTableITCase.java | 2 +-
.../store/connector/ReadWriteTableITCase.java | 726 ++---------
.../store/connector/ReadWriteTableTestBase.java | 387 ++++++
.../store/connector/ReadWriteTableTestUtil.java | 534 ++++++++
.../table/store/connector/ShowCreateUtil.java | 153 ++-
.../store/connector/TableStoreFactoryTest.java | 10 +-
.../table/store/connector/TableStoreTestBase.java | 37 +-
9 files changed, 2505 insertions(+), 632 deletions(-)
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
new file mode 100644
index 0000000..a0a6b3f
--- /dev/null
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
@@ -0,0 +1,1286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.connector;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.table.api.EnvironmentSettings;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.table.store.file.utils.BlockingIterator;
+import org.apache.flink.types.Row;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyExchangeRates;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyExchangeRatesChangelogWithoutUB;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithUB;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithoutUB;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.exchangeRatesChangelogWithoutUB;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyExchangeRates;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyExchangeRatesChangelogWithoutUB;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyRates;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.hourlyRatesChangelogWithoutUB;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Table store IT case when the managed table has composite primary keys and multiple partition
+ * fields.
+ */
+public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTestBase {
+
+ @Test
+ public void testBatchWriteWithMultiPartitionedRecordsWithMultiPk() throws Exception {
+ // input is hourlyExchangeRates()
+ List<Row> expectedRecords =
+ Arrays.asList(
+ // to_currency is USD, dt = 2022-01-01, hh = 11
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01", "11"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "11"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
+ // to_currency is USD, dt = 2022-01-01, hh = 12
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
+ changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
+ changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12"),
+ // to_currency is Euro, dt = 2022-01-02, hh = 23
+ changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23"));
+ // test batch read
+ String managedTable =
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ null,
+ Collections.emptyList(),
+ expectedRecords);
+
+ checkFileStorePath(tEnv, managedTable, hourlyExchangeRates().f1);
+
+ // test streaming read
+ final StreamTableEnvironment streamTableEnv =
+ StreamTableEnvironment.create(buildStreamEnv());
+ registerTable(streamTableEnv, managedTable);
+ BlockingIterator<Row, Row> streamIter =
+ collectAndCheck(
+ streamTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ expectedRecords);
+
+ // overwrite static partition dt = 2022-01-02 and hh = 23
+ Map<String, String> overwritePartition = new LinkedHashMap<>();
+ overwritePartition.put("dt", "'2022-01-02'");
+ overwritePartition.put("hh", "'23'");
+ prepareEnvAndOverwrite(
+ managedTable,
+ overwritePartition,
+ Collections.singletonList(new String[] {"'US Dollar'", "'Thai Baht'", "33.51"}));
+
+ // streaming iter will not receive any changelog
+ assertNoMoreRecords(streamIter);
+
+ // batch read to check partition refresh
+ collectAndCheck(
+ tEnv,
+ managedTable,
+ Collections.emptyMap(),
+ "dt = '2022-01-02' AND hh = '23'",
+ Collections.singletonList(
+ changelogRow(
+ "+I",
+ "US Dollar",
+ "Thai Baht",
+ 33.51d,
+ "2022-01-02",
+ "23")))
+ .close();
+
+ // test partition filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "dt = '2022-01-01'",
+ Collections.emptyList(),
+ Arrays.asList(
+ // to_currency is USD, dt = 2022-01-01, hh = 11
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01", "11"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "11"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
+ // to_currency is USD, dt = 2022-01-01, hh = 12
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
+ changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
+ changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12")));
+
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "dt = '2022-01-01' AND hh = '12'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
+ changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
+ changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12")));
+
+ // test field filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "to_currency = 'Euro'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")));
+
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "from_currency = 'HK Dollar'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12")));
+
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "rate_by_to_currency > 0.5",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01", "11"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
+ changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
+ changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")));
+
+ // test partition and field filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "rate_by_to_currency > 0.9 AND hh = '23'",
+ Collections.emptyList(),
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23")));
+
+ // test projection
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ null,
+ Arrays.asList("from_currency", "dt", "hh"),
+ Arrays.asList(
+ // to_currency is USD, dt = 2022-01-01, hh = 11
+ changelogRow("+I", "Euro", "2022-01-01", "11"),
+ changelogRow("+I", "HK Dollar", "2022-01-01", "11"),
+ changelogRow("+I", "Singapore Dollar", "2022-01-01", "11"),
+ changelogRow("+I", "Yen", "2022-01-01", "11"),
+ changelogRow("+I", "US Dollar", "2022-01-01", "11"),
+ // to_currency is USD, dt = 2022-01-01, hh = 12
+ changelogRow("+I", "US Dollar", "2022-01-01", "12"),
+ changelogRow("+I", "Euro", "2022-01-01", "12"),
+ changelogRow("+I", "HK Dollar", "2022-01-01", "12"),
+ changelogRow("+I", "Singapore Dollar", "2022-01-01", "12"),
+ changelogRow("+I", "Yen", "2022-01-01", "12"),
+ // to_currency is Euro, dt = 2022-01-02, hh = 23
+ changelogRow("+I", "US Dollar", "2022-01-02", "23"),
+ changelogRow("+I", "Singapore Dollar", "2022-01-02", "23")));
+
+ // test projection and filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "dt = '2022-01-01' AND hh >= '12' OR rate_by_to_currency > 2",
+ Arrays.asList("from_currency", "to_currency"),
+ Arrays.asList(
+ // to_currency is USD, dt = 2022-01-01, hh = 12
+ changelogRow("+I", "US Dollar", "US Dollar"),
+ changelogRow("+I", "Euro", "US Dollar"),
+ changelogRow("+I", "HK Dollar", "US Dollar"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar"),
+ changelogRow("+I", "Yen", "US Dollar")));
+ }
+
+ @Test
+ public void testBatchWriteWithSinglePartitionedRecordsWithMultiPk() throws Exception {
+ // input is dailyExchangeRates()
+ List<Row> expectedRecords =
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-01"),
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-01"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02"));
+ // test batch read
+ String managedTable =
+ collectAndCheckBatchReadWrite(
+ Collections.singletonList("dt"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ null,
+ Collections.emptyList(),
+ expectedRecords);
+
+ checkFileStorePath(tEnv, managedTable, dailyExchangeRates().f1);
+
+ // test streaming read
+ final StreamTableEnvironment streamTableEnv =
+ StreamTableEnvironment.create(buildStreamEnv());
+ registerTable(streamTableEnv, managedTable);
+ BlockingIterator<Row, Row> streamIter =
+ collectAndCheck(
+ streamTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ expectedRecords);
+
+ // overwrite dynamic partition
+ prepareEnvAndOverwrite(
+ managedTable,
+ Collections.emptyMap(),
+ Collections.singletonList(
+ new String[] {"'US Dollar'", "'Thai Baht'", "33.51", "'2022-01-01'"}));
+
+ // streaming iter will not receive any changelog
+ assertNoMoreRecords(streamIter);
+
+ // batch read to check partition refresh
+ collectAndCheck(
+ tEnv,
+ managedTable,
+ Collections.emptyMap(),
+ "dt = '2022-01-01'",
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "Thai Baht", 33.51d, "2022-01-01")))
+ .close();
+
+ // test partition filter
+ collectAndCheckBatchReadWrite(
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ "dt = '2022-01-02'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02")));
+
+ // test field filter
+ collectAndCheckBatchReadWrite(
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ "rate_by_to_currency < 0.1",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02")));
+
+ // test partition and field filter
+ collectAndCheckBatchReadWrite(
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ "rate_by_to_currency > 0.9 OR dt = '2022-01-02'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-01"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02")));
+
+ // test projection
+ collectAndCheckBatchReadWrite(
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ null,
+ Arrays.asList("rate_by_to_currency", "dt"),
+ Arrays.asList(
+ changelogRow("+I", 1.0d, "2022-01-01"), // US Dollar,US Dollar
+ changelogRow("+I", 1.11d, "2022-01-01"), // Euro,US Dollar
+ changelogRow("+I", 0.13d, "2022-01-01"), // HK Dollar,US Dollar
+ changelogRow("+I", 0.0082d, "2022-01-01"), // Yen,US Dollar
+ changelogRow("+I", 0.74d, "2022-01-01"), // Singapore Dollar,US Dollar
+ changelogRow("+I", 0.9d, "2022-01-01"), // US Dollar,Euro
+ changelogRow("+I", 0.67d, "2022-01-01"), // Singapore Dollar,Euro
+ changelogRow("+I", 1.0d, "2022-01-01"), // Yen,Yen
+ changelogRow("+I", 19.25d, "2022-01-01"), // Chinese Yuan,Yen
+ changelogRow("+I", 122.46d, "2022-01-01"), // Singapore Dollar,Yen
+ changelogRow("+I", 0.0081d, "2022-01-02"), // Yen,US Dollar
+ changelogRow("+I", 1.0d, "2022-01-02") // US Dollar,US Dollar
+ ));
+
+ // test projection and filter
+ collectAndCheckBatchReadWrite(
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ "dt = '2022-01-02' OR rate_by_to_currency > 100",
+ Arrays.asList("from_currency", "to_currency"),
+ Arrays.asList(
+ changelogRow("+I", "Yen", "US Dollar"),
+ changelogRow("+I", "US Dollar", "US Dollar"),
+ changelogRow("+I", "Singapore Dollar", "Yen")));
+ }
+
+ @Test
+ public void testBatchWriteWithNonPartitionedRecordsWithMultiPk() throws Exception {
+ // input is exchangeRates()
+ List<Row> expectedRecords =
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "Euro", "US Dollar", 1.11d),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+ changelogRow("+I", "Singapore Dollar", "Yen", 122.46d));
+ // test batch read
+ String managedTable =
+ collectAndCheckBatchReadWrite(
+ Collections.emptyList(), // partition
+ Arrays.asList("from_currency", "to_currency"), // pk
+ null,
+ Collections.emptyList(),
+ expectedRecords);
+
+ checkFileStorePath(tEnv, managedTable, null);
+
+ // test streaming read
+ final StreamTableEnvironment streamTableEnv =
+ StreamTableEnvironment.create(buildStreamEnv());
+ registerTable(streamTableEnv, managedTable);
+ BlockingIterator<Row, Row> streamIter =
+ collectAndCheck(
+ streamTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ expectedRecords);
+
+ // overwrite the whole table
+ prepareEnvAndOverwrite(
+ managedTable,
+ Collections.emptyMap(),
+ Collections.singletonList(new String[] {"'US Dollar'", "'Thai Baht'", "33.51"}));
+
+ // streaming iter will not receive any changelog
+ assertNoMoreRecords(streamIter);
+
+ // batch read to check data refresh
+ collectAndCheck(
+ tEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "Thai Baht", 33.51d)))
+ .close();
+
+ // test field filter
+ collectAndCheckBatchReadWrite(
+ Collections.emptyList(), // partition
+ Arrays.asList("from_currency", "to_currency"), // pk
+ "rate_by_to_currency < 0.1",
+ Collections.emptyList(),
+ Collections.singletonList(changelogRow("+I", "Yen", "US Dollar", 0.0081d)));
+
+ // test projection
+ collectAndCheckBatchReadWrite(
+ Collections.emptyList(), // partition
+ Arrays.asList("from_currency", "to_currency"), // pk
+ null,
+ Collections.singletonList("rate_by_to_currency"),
+ Arrays.asList(
+ changelogRow("+I", 1.11d), // Euro, US Dollar
+ changelogRow("+I", 0.13d), // HK Dollar, US Dollar
+ changelogRow("+I", 0.74d), // Singapore Dollar, US Dollar
+ changelogRow("+I", 0.0081d), // Yen, US Dollar
+ changelogRow("+I", 1.0d), // US Dollar, US Dollar
+ changelogRow("+I", 0.9d), // US Dollar, Euro
+ changelogRow("+I", 0.67d), // Singapore Dollar, Euro
+ changelogRow("+I", 1.0d), // Yen, Yen
+ changelogRow("+I", 19.25d), // Chinese Yuan, Yen
+ changelogRow("+I", 122.46d) // Singapore Dollar, Yen
+ ));
+
+ // test projection and filter
+ collectAndCheckBatchReadWrite(
+ Collections.emptyList(), // partition
+ Arrays.asList("from_currency", "to_currency"), // pk
+ "rate_by_to_currency > 100",
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.singletonList(changelogRow("+I", "Singapore Dollar", "Yen")));
+ }
+
+ @Test
+ public void testBatchWriteMultiPartitionedRecordsWithOnePk() throws Exception {
+ // input is hourlyRates()
+ List<Row> expectedRecords =
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 00
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "00"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "00"),
+ // dt = 2022-01-01, hh = 20
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
+ // dt = 2022-01-02, hh = 12
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "12"));
+ // test batch read
+ String managedTable =
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ null,
+ Collections.emptyList(),
+ expectedRecords);
+
+ checkFileStorePath(tEnv, managedTable, hourlyRates().f1);
+
+ // test streaming read
+ final StreamTableEnvironment streamTableEnv =
+ StreamTableEnvironment.create(buildStreamEnv());
+ registerTable(streamTableEnv, managedTable);
+ BlockingIterator<Row, Row> streamIter =
+ collectAndCheck(
+ streamTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ expectedRecords);
+
+ // dynamic overwrite
+ // INSERT OVERWRITE `manged_table-${uuid}` VALUES(...)
+ prepareEnvAndOverwrite(
+ managedTable,
+ Collections.emptyMap(),
+ Collections.singletonList(
+ new String[] {"'HK Dollar'", "80", "'2022-01-01'", "'00'"}));
+
+ // INSERT OVERWRITE `manged_table-${uuid}` PARTITION (dt = '2022-01-02') VALUES(...)
+ prepareEnvAndOverwrite(
+ managedTable,
+ Collections.singletonMap("dt", "'2022-01-02'"),
+ Collections.singletonList(new String[] {"'Euro'", "120", "'12'"}));
+
+ // batch read to check data refresh
+ collectAndCheck(
+ tEnv,
+ managedTable,
+ Collections.emptyMap(),
+ "hh = '00' OR hh = '12'",
+ Arrays.asList(
+ changelogRow("+I", "HK Dollar", 80L, "2022-01-01", "00"),
+ changelogRow("+I", "Euro", 120L, "2022-01-02", "12")));
+
+ // streaming iter will not receive any changelog
+ assertNoMoreRecords(streamIter);
+
+ // test partition filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ "hh >= '10'",
+ Collections.emptyList(),
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 20
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
+ // dt = 2022-01-02, hh = 12
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+
+ // test field filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ "rate >= 119",
+ Collections.emptyList(),
+ Collections.singletonList(
+ // dt = 2022-01-02, hh = 12
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+
+ // test projection
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ null,
+ Collections.singletonList("hh"),
+ Arrays.asList(
+ changelogRow("+I", "00"), // Yen, 1L, 2022-01-01
+ changelogRow("+I", "00"), // Euro, 114L, 2022-01-01
+ changelogRow("+I", "00"), // US Dollar, 114L, 2022-01-01
+ changelogRow("+I", "20"), // Yen, 1L, 2022-01-01
+ changelogRow("+I", "20"), // Euro, 114L, 2022-01-01
+ changelogRow("+I", "20"), // US Dollar, 114L, 2022-01-01
+ changelogRow("+I", "12") // Euro, 119L, "2022-01-02
+ ));
+
+ // test projection and filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ "rate > 100 AND hh >= '20'",
+ Collections.singletonList("rate"),
+ Collections.singletonList(changelogRow("+I", 114L)));
+ }
+
+ @Test
+ public void testBatchWriteMultiPartitionedRecordsWithoutPk() throws Exception {
+ // input is hourlyRates()
+
+ // test batch read
+ String managedTable =
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ null,
+ Collections.emptyList(),
+ hourlyRates().f0);
+
+ checkFileStorePath(tEnv, managedTable, hourlyRates().f1);
+
+ // test streaming read
+ final StreamTableEnvironment streamTableEnv =
+ StreamTableEnvironment.create(buildStreamEnv());
+ registerTable(streamTableEnv, managedTable);
+ BlockingIterator<Row, Row> streamIter =
+ collectAndCheck(
+ streamTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ hourlyRates().f0);
+
+ // dynamic overwrite the whole table with null partitions
+ String query =
+ String.format(
+ "INSERT OVERWRITE `%s` SELECT 'Yen', 1, CAST(null AS STRING), CAST(null AS STRING) FROM `%s`",
+ managedTable, managedTable);
+ prepareEnvAndOverwrite(managedTable, query);
+ // check new partition path
+ checkFileStorePath(tEnv, managedTable, "dt:null,hh:null");
+
+ // batch read to check data refresh
+ collectAndCheck(
+ tEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ Collections.singletonList(changelogRow("+I", "Yen", 1L, null, null)));
+
+ // streaming iter will not receive any changelog
+ assertNoMoreRecords(streamIter);
+
+ // test partition filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ "hh >= '10'",
+ Collections.emptyList(),
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 20
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
+ // dt = 2022-01-02, hh = 12
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+
+ // test field filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ "rate >= 119",
+ Collections.emptyList(),
+ Collections.singletonList(
+ // dt = 2022-01-02, hh = 12
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "12")));
+
+ // test projection
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ null,
+ Collections.singletonList("currency"),
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 00
+ changelogRow("+I", "US Dollar"),
+ changelogRow("+I", "Euro"),
+ changelogRow("+I", "Yen"),
+ changelogRow("+I", "Euro"),
+ changelogRow("+I", "US Dollar"),
+ // dt = 2022-01-01, hh = 20
+ changelogRow("+I", "US Dollar"),
+ changelogRow("+I", "Euro"),
+ changelogRow("+I", "Yen"),
+ changelogRow("+I", "Euro"),
+ changelogRow("+I", "US Dollar"),
+ // dt = 2022-01-02, hh = 12
+ changelogRow("+I", "Euro")));
+
+ // test projection and filter
+ collectAndCheckBatchReadWrite(
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ "rate > 100 AND hh >= '20'",
+ Collections.singletonList("rate"),
+ Collections.singletonList(changelogRow("+I", 114L)));
+ }
+
+ @Test
+ public void testEnableLogAndStreamingReadWriteSinglePartitionedRecordsWithMultiPk()
+ throws Exception {
+ // input is dailyExchangeRatesChangelogWithoutUB()
+ // test hybrid read
+ Tuple2<String, BlockingIterator<Row, Row>> tuple =
+ collectAndCheckStreamingReadWriteWithoutClose(
+ Collections.singletonList("dt"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.emptyMap(),
+ null,
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+ changelogRow(
+ "+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-02")));
+ String managedTable = tuple.f0;
+ checkFileStorePath(tEnv, managedTable, dailyRatesChangelogWithoutUB().f1);
+ BlockingIterator<Row, Row> streamIter = tuple.f1;
+
+ // test streaming consume changelog
+ tEnv.executeSql(
+ String.format(
+ "INSERT INTO `%s` PARTITION (dt = '2022-01-03')\n"
+ + "VALUES('Chinese Yuan', 'HK Dollar', 1.231)\n",
+ managedTable))
+ .await();
+
+ assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
+ .containsExactlyInAnyOrderElementsOf(
+ Collections.singletonList(
+ changelogRow(
+ "+I", "Chinese Yuan", "HK Dollar", 1.231d, "2022-01-03")));
+
+ // dynamic overwrite the whole table
+ String query =
+ String.format(
+ "INSERT OVERWRITE `%s` SELECT 'US Dollar', 'US Dollar', 1, '2022-04-02' FROM `%s`",
+ managedTable, managedTable);
+ prepareEnvAndOverwrite(managedTable, query);
+ checkFileStorePath(tEnv, managedTable, "dt:2022-04-02");
+
+ // batch read to check data refresh
+ final StreamTableEnvironment batchTableEnv =
+ StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
+ registerTable(batchTableEnv, managedTable);
+ collectAndCheck(
+ batchTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-04-02")));
+
+ // check no changelog generated for streaming read
+ assertNoMoreRecords(streamIter);
+
+ // filter on partition and field filter
+ collectAndCheckStreamingReadWriteWithClose(
+ true,
+ Collections.singletonList("dt"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.emptyMap(),
+ "dt = '2022-01-02' AND from_currency = 'US Dollar'",
+ Collections.emptyList(),
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02")));
+
+ // test projection and filter
+ collectAndCheckStreamingReadWriteWithClose(
+ true,
+ Collections.singletonList("dt"),
+ Arrays.asList("from_currency", "to_currency", "dt"),
+ Collections.emptyMap(),
+ "dt = '2022-01-01' AND rate_by_to_currency IS NULL",
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.emptyList());
+ }
+
+ @Test
+ public void testEnableLogAndStreamingReadWriteMultiPartitionedRecordsWithMultiPk()
+ throws Exception {
+ // input is hourlyExchangeRatesChangelogWithoutUB()
+ // test hybrid read
+ Tuple2<String, BlockingIterator<Row, Row>> tuple =
+ collectAndCheckStreamingReadWriteWithoutClose(
+ Arrays.asList("dt", "hh"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Collections.emptyMap(),
+ null,
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow(
+ "+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
+ changelogRow(
+ "+I",
+ "Singapore Dollar",
+ "US Dollar",
+ 0.76d,
+ "2022-01-02",
+ "20"),
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ changelogRow(
+ "+I", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
+ changelogRow(
+ "+I", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
+ changelogRow(
+ "+I",
+ "Singapore Dollar",
+ "Yen",
+ 90.1d,
+ "2022-01-02",
+ "20")));
+ String managedTable = tuple.f0;
+ checkFileStorePath(tEnv, managedTable, hourlyExchangeRatesChangelogWithoutUB().f1);
+ BlockingIterator<Row, Row> streamIter = tuple.f1;
+
+ // test streaming consume changelog
+ tEnv.executeSql(
+ String.format(
+ "INSERT INTO `%s` \n"
+ + "VALUES('Chinese Yuan', 'HK Dollar', 1.231, '2022-01-03', '15')\n",
+ managedTable))
+ .await();
+
+ assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
+ .containsExactlyInAnyOrderElementsOf(
+ Collections.singletonList(
+ changelogRow(
+ "+I",
+ "Chinese Yuan",
+ "HK Dollar",
+ 1.231d,
+ "2022-01-03",
+ "15")));
+
+ // dynamic overwrite the whole table
+ String query =
+ String.format(
+ "INSERT OVERWRITE `%s` SELECT 'US Dollar', 'US Dollar', 1, '2022-04-02', '10' FROM `%s`",
+ managedTable, managedTable);
+ prepareEnvAndOverwrite(managedTable, query);
+ checkFileStorePath(tEnv, managedTable, "dt:2022-04-02,hh:10");
+
+ // batch read to check data refresh
+ final StreamTableEnvironment batchTableEnv =
+ StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
+ registerTable(batchTableEnv, managedTable);
+ collectAndCheck(
+ batchTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-04-02", "10")));
+
+ // check no changelog generated for streaming read
+ assertNoMoreRecords(streamIter);
+
+ // filter on partition and field filter
+ collectAndCheckStreamingReadWriteWithClose(
+ true,
+ Arrays.asList("dt", "hh"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Collections.emptyMap(),
+ "dt = '2022-01-02' AND from_currency = 'US Dollar'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
+
+ // test projection and filter
+ collectAndCheckStreamingReadWriteWithClose(
+ true,
+ Arrays.asList("dt", "hh"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"),
+ Collections.emptyMap(),
+ "dt = '2022-01-02' AND from_currency = 'US Dollar'",
+ Arrays.asList("from_currency", "to_currency"),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "Euro"),
+ changelogRow("+I", "US Dollar", "Yen")));
+ }
+
+ @Test
+ public void testEnableLogAndStreamingReadWriteMultiPartitionedRecordsWithoutPk()
+ throws Exception {
+ // input is dailyExchangeRatesChangelogWithUB()
+ // test hybrid read
+ Tuple2<String, BlockingIterator<Row, Row>> tuple =
+ collectAndCheckStreamingReadWriteWithoutClose(
+ Collections.singletonList("dt"),
+ Collections.emptyList(),
+ Collections.emptyMap(),
+ null,
+ Collections.emptyList(),
+ Arrays.asList(
+ // dt = 2022-01-01
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
+ // dt = 2022-01-02
+ changelogRow("+I", "Euro", 115L, "2022-01-02")));
+ String managedTable = tuple.f0;
+ checkFileStorePath(tEnv, managedTable, dailyRatesChangelogWithUB().f1);
+ BlockingIterator<Row, Row> streamIter = tuple.f1;
+
+ // test streaming consume changelog
+ tEnv.executeSql(
+ String.format(
+ "INSERT INTO `%s` PARTITION (dt = '2022-04-02')\n"
+ + "VALUES('Euro', 116)\n",
+ managedTable))
+ .await();
+
+ assertThat(streamIter.collect(1, 5, TimeUnit.SECONDS))
+ .containsExactlyInAnyOrderElementsOf(
+ Collections.singletonList(changelogRow("+I", "Euro", 116L, "2022-04-02")));
+
+ // dynamic overwrite the whole table
+ String query =
+ String.format(
+ "INSERT OVERWRITE `%s` SELECT 'US Dollar', 103, '2022-04-02' FROM `%s`",
+ managedTable, managedTable);
+ prepareEnvAndOverwrite(managedTable, query);
+ checkFileStorePath(tEnv, managedTable, "dt:2022-04-02");
+
+ // batch read to check data refresh
+ final StreamTableEnvironment batchTableEnv =
+ StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
+ registerTable(batchTableEnv, managedTable);
+ collectAndCheck(
+ batchTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ Collections.singletonList(changelogRow("+I", "US Dollar", 103L, "2022-04-02")));
+
+ // check no changelog generated for streaming read
+ assertNoMoreRecords(streamIter);
+
+ // filter on partition and field filter
+ collectAndCheckStreamingReadWriteWithClose(
+ true,
+ Collections.singletonList("dt"),
+ Collections.emptyList(),
+ Collections.emptyMap(),
+ "dt = '2022-01-01' AND currency = 'Yen'",
+ Collections.emptyList(),
+ Collections.emptyList());
+
+ // test projection and filter
+ collectAndCheckStreamingReadWriteWithClose(
+ true,
+ Collections.singletonList("dt"),
+ Collections.emptyList(),
+ Collections.emptyMap(),
+ "dt = '2022-01-01' AND rate = 103",
+ Collections.singletonList("currency"),
+ Collections.emptyList());
+ }
+
+ @Test
+ public void testReadLatestChangelogOfMultiPartitionedRecordsWithMultiPk() throws Exception {
+ // input is hourlyExchangeRatesChangelogWithoutUB()
+ List<Row> expectedRecords = new ArrayList<>(hourlyExchangeRatesChangelogWithoutUB().f0);
+ expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"));
+ expectedRecords.add(
+ changelogRow("-U", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"));
+ expectedRecords.add(
+ changelogRow("-U", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"));
+ expectedRecords.add(changelogRow("-U", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"));
+ expectedRecords.add(
+ changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"));
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ null,
+ Collections.emptyList(),
+ expectedRecords);
+
+ // test partition filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "dt = '2022-01-02' AND hh = '21'",
+ Collections.emptyList(),
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
+
+ // test field filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "rate_by_to_currency IS NOT NULL AND from_currency = 'US Dollar'",
+ Collections.emptyList(),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ changelogRow("-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ // to_currency is Yen
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
+
+ // test partition and field filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "hh = '21' AND from_currency = 'US Dollar'",
+ Collections.emptyList(),
+ Collections.singletonList(
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21")));
+
+ // test projection
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ null,
+ Arrays.asList("from_currency", "to_currency"),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar"),
+ changelogRow("+I", "Euro", "US Dollar"),
+ changelogRow("+I", "HK Dollar", "US Dollar"),
+ changelogRow("+I", "Yen", "US Dollar"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar"),
+ changelogRow("-D", "US Dollar", "US Dollar"),
+ changelogRow("-D", "Euro", "US Dollar"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro"),
+ changelogRow("+I", "Singapore Dollar", "Euro"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen"),
+ changelogRow("+I", "Chinese Yuan", "Yen"),
+ changelogRow("+I", "Singapore Dollar", "Yen"),
+ changelogRow("+I", "US Dollar", "Yen")));
+
+ // test projection and filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"),
+ Arrays.asList("from_currency", "to_currency", "dt", "hh"), // pk
+ "rate_by_to_currency > 100",
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.singletonList(changelogRow("+I", "US Dollar", "Yen")));
+ }
+
+ @Test
+ public void testReadLatestChangelogOfSinglePartitionedRecordsWithMultiPk() throws Exception {
+ // input is dailyExchangeRatesChangelogWithoutUB()
+ List<Row> expectedRecords = new ArrayList<>(dailyExchangeRatesChangelogWithoutUB().f0);
+ expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"));
+ expectedRecords.add(changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01"));
+ expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"));
+
+ collectLatestLogAndCheck(
+ false,
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ null,
+ Collections.emptyList(),
+ expectedRecords);
+
+ // test partition filter
+ collectLatestLogAndCheck(
+ false,
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ "dt = '2022-01-02'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02"),
+ changelogRow("-U", "Singapore Dollar", "Yen", 90.32d, "2022-01-02")));
+
+ // test field filter
+ collectLatestLogAndCheck(
+ false,
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ "rate_by_to_currency IS NULL",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
+ changelogRow("-U", "Euro", "US Dollar", null, "2022-01-01"),
+ changelogRow("-U", "US Dollar", "US Dollar", null, "2022-01-01")));
+
+ // test partition and field filter
+ collectLatestLogAndCheck(
+ false,
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ "dt = '2022-01-02' AND from_currency = 'Yen'",
+ Collections.emptyList(),
+ Collections.singletonList(changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02")));
+
+ // test projection and filter
+ collectLatestLogAndCheck(
+ false,
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ "rate_by_to_currency > 100",
+ Arrays.asList("from_currency", "to_currency"),
+ Collections.singletonList(changelogRow("+U", "Singapore Dollar", "Yen")));
+ }
+
+ @Test
+ public void testReadLatestChangelogOfNonPartitionedRecordsWithMultiPk() throws Exception {
+ // input is exchangeRatesChangelogWithoutUB()
+ List<Row> expectedRecords = new ArrayList<>(exchangeRatesChangelogWithoutUB());
+ expectedRecords.add(changelogRow("-U", "Yen", "US Dollar", 0.0082d));
+ expectedRecords.add(changelogRow("-U", "Euro", "US Dollar", 1.11d));
+ expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Euro", 0.67d));
+ expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 90.32d));
+ expectedRecords.add(changelogRow("-U", "Singapore Dollar", "Yen", 122.46d));
+
+ collectLatestLogAndCheck(
+ false,
+ Collections.emptyList(), // partition
+ Arrays.asList("from_currency", "to_currency"), // pk
+ null,
+ Collections.emptyList(),
+ expectedRecords);
+
+ // test field filter
+ collectLatestLogAndCheck(
+ false,
+ Collections.emptyList(), // partition
+ Arrays.asList("from_currency", "to_currency"), // pk
+ "rate_by_to_currency < 1 OR rate_by_to_currency > 100",
+ Collections.emptyList(),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d),
+ changelogRow("-U", "Yen", "US Dollar", 0.0082d),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+ changelogRow("+U", "Yen", "US Dollar", 0.0081d),
+ changelogRow("-D", "Yen", "US Dollar", 0.0081d),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+ changelogRow("-U", "Singapore Dollar", "Euro", 0.67d),
+ changelogRow("+U", "Singapore Dollar", "Euro", 0.69d),
+ // to_currency is Yen
+ changelogRow("+U", "Singapore Dollar", "Yen", 122.46d),
+ changelogRow("-U", "Singapore Dollar", "Yen", 122.46d),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122d)));
+
+ // test projection and filter
+ collectLatestLogAndCheck(
+ false,
+ Collections.emptyList(), // partition
+ Arrays.asList("from_currency", "to_currency"), // pk
+ "rate_by_to_currency < 1 OR rate_by_to_currency > 100",
+ Arrays.asList("from_currency", "to_currency"),
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "HK Dollar", "US Dollar"),
+ changelogRow("+I", "Yen", "US Dollar"),
+ changelogRow("-U", "Yen", "US Dollar"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar"),
+ changelogRow("+U", "Yen", "US Dollar"),
+ changelogRow("-D", "Yen", "US Dollar"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro"),
+ changelogRow("+I", "Singapore Dollar", "Euro"),
+ changelogRow("-U", "Singapore Dollar", "Euro"),
+ changelogRow("+U", "Singapore Dollar", "Euro"),
+ // to_currency is Yen
+ changelogRow("+U", "Singapore Dollar", "Yen"),
+ changelogRow("-U", "Singapore Dollar", "Yen"),
+ changelogRow("+U", "Singapore Dollar", "Yen")));
+ }
+
+ @Test
+ public void testReadLatestChangelogOfMultiPartitionedRecordsWithOnePk() throws Exception {
+ // input is hourlyRatesChangelogWithoutUB()
+ List<Row> expectedRecords = new ArrayList<>(hourlyRatesChangelogWithoutUB().f0);
+ expectedRecords.remove(changelogRow("+U", "Euro", 119L, "2022-01-02", "23"));
+ expectedRecords.add(changelogRow("-U", "Euro", 114L, "2022-01-01", "15"));
+
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ null,
+ Collections.emptyList(),
+ expectedRecords);
+
+ // test partition filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ "dt >= '2022-01-02'",
+ Collections.emptyList(),
+ Collections.singletonList(changelogRow("+I", "Euro", 119L, "2022-01-02", "23")));
+
+ // test field filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ "rate = 1",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
+
+ // test projection and filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ "rate = 1",
+ Collections.singletonList("currency"),
+ Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")));
+ }
+
+ @Test
+ public void testReadLatestChangelogOfMultiPartitionedRecordsWithoutPk() throws Exception {
+ // input is hourlyRatesChangelogWithUB()
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ null,
+ Collections.emptyList(),
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 15
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+ // dt = 2022-01-02, hh = 20
+ changelogRow("+I", "Euro", 114L, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", 114L, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", 119L, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", 115L, "2022-01-02", "20")));
+
+ // test partition filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ "hh <> '20'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
+
+ // test field filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ "rate = 1",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15")));
+
+ // test projection and filter
+ collectLatestLogAndCheck(
+ false,
+ Arrays.asList("dt", "hh"), // partition
+ Collections.emptyList(), // pk
+ "rate = 1",
+ Collections.singletonList("currency"),
+ Arrays.asList(changelogRow("+I", "Yen"), changelogRow("-D", "Yen")));
+ }
+}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CreateTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CreateTableITCase.java
index 26f3c1c..cbbe4b4 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CreateTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CreateTableITCase.java
@@ -56,7 +56,7 @@ public class CreateTableITCase extends TableStoreTestBase {
Collections.emptyMap(),
RowType.of(new IntType(), new VarCharType()),
Collections.emptyList(),
- new int[0]);
+ Collections.emptyList());
public CreateTableITCase(
RuntimeExecutionMode executionMode,
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/DropTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/DropTableITCase.java
index eb65439..89631e4 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/DropTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/DropTableITCase.java
@@ -54,7 +54,7 @@ public class DropTableITCase extends TableStoreTestBase {
Collections.emptyMap(),
RowType.of(new IntType(), new VarCharType()),
Collections.emptyList(),
- new int[0]);
+ Collections.emptyList());
public DropTableITCase(
RuntimeExecutionMode executionMode,
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
index 88b608f..535f097 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
@@ -18,69 +18,53 @@
package org.apache.flink.table.store.connector;
-import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.dag.Transformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSink;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.transformations.PartitionTransformation;
-import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.EnvironmentSettings;
-import org.apache.flink.table.api.Schema;
-import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.table.catalog.CatalogBaseTable;
-import org.apache.flink.table.catalog.CatalogTable;
-import org.apache.flink.table.catalog.Column;
import org.apache.flink.table.catalog.ObjectIdentifier;
-import org.apache.flink.table.catalog.ObjectPath;
-import org.apache.flink.table.catalog.ResolvedCatalogTable;
-import org.apache.flink.table.catalog.ResolvedSchema;
import org.apache.flink.table.connector.sink.DataStreamSinkProvider;
import org.apache.flink.table.connector.sink.DynamicTableSink;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.factories.DynamicTableFactory.Context;
+import org.apache.flink.table.factories.DynamicTableFactory;
import org.apache.flink.table.factories.FactoryUtil;
import org.apache.flink.table.runtime.connector.sink.SinkRuntimeProviderContext;
import org.apache.flink.table.store.connector.sink.TableStoreSink;
-import org.apache.flink.table.store.file.FileStoreOptions;
import org.apache.flink.table.store.file.utils.BlockingIterator;
-import org.apache.flink.table.store.kafka.KafkaTableTestBase;
-import org.apache.flink.table.store.log.LogOptions;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.VarCharType;
import org.apache.flink.types.Row;
import org.junit.Test;
import javax.annotation.Nullable;
-import java.nio.file.Paths;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
-import static org.apache.flink.table.planner.factories.TestValuesTableFactory.registerData;
-import static org.apache.flink.table.store.connector.TableStoreFactoryOptions.LOG_SYSTEM;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRates;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithUB;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithoutUB;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.rates;
+import static org.apache.flink.table.store.connector.ShowCreateUtil.buildSimpleSelectQuery;
import static org.apache.flink.table.store.connector.TableStoreFactoryOptions.SCAN_PARALLELISM;
import static org.apache.flink.table.store.connector.TableStoreFactoryOptions.SINK_PARALLELISM;
-import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
-import static org.apache.flink.table.store.log.LogOptions.LOG_PREFIX;
+import static org.apache.flink.table.store.connector.TableStoreTestBase.createResolvedTable;
import static org.assertj.core.api.Assertions.assertThat;
/** IT cases for managed table dml. */
-public class ReadWriteTableITCase extends KafkaTableTestBase {
-
- private String rootPath;
+public class ReadWriteTableITCase extends ReadWriteTableTestBase {
@Test
public void testBatchWriteWithPartitionedRecordsWithPk() throws Exception {
@@ -97,7 +81,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
String managedTable =
collectAndCheckBatchReadWrite(
true, true, null, Collections.emptyList(), expectedRecords);
- checkFileStorePath(tEnv, managedTable);
+ checkFileStorePath(tEnv, managedTable, dailyRates().f1);
// test streaming read
final StreamTableEnvironment streamTableEnv =
@@ -105,7 +89,11 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
registerTable(streamTableEnv, managedTable);
BlockingIterator<Row, Row> streamIter =
collectAndCheck(
- streamTableEnv, managedTable, Collections.emptyMap(), expectedRecords);
+ streamTableEnv,
+ managedTable,
+ Collections.emptyMap(),
+ null,
+ expectedRecords);
// overwrite static partition 2022-01-02
prepareEnvAndOverwrite(
@@ -121,12 +109,9 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
tEnv,
managedTable,
Collections.emptyMap(),
+ "dt = '2022-01-02'",
Arrays.asList(
// part = 2022-01-01
- changelogRow("+I", "US Dollar", 114L, "2022-01-01"),
- changelogRow("+I", "Yen", 1L, "2022-01-01"),
- changelogRow("+I", "Euro", 114L, "2022-01-01"),
- // part = 2022-01-02
changelogRow("+I", "Euro", 100L, "2022-01-02"),
changelogRow("+I", "Yen", 1L, "2022-01-02")))
.close();
@@ -206,8 +191,8 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
// test batch read
String managedTable =
collectAndCheckBatchReadWrite(
- true, false, null, Collections.emptyList(), dailyRates());
- checkFileStorePath(tEnv, managedTable);
+ true, false, null, Collections.emptyList(), dailyRates().f0);
+ checkFileStorePath(tEnv, managedTable, dailyRates().f1);
// overwrite dynamic partition
prepareEnvAndOverwrite(
@@ -225,6 +210,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
streamTableEnv,
managedTable,
Collections.emptyMap(),
+ null,
Arrays.asList(
// part = 2022-01-01
changelogRow("+I", "Euro", 90L, "2022-01-01"),
@@ -234,7 +220,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
// test partition filter
collectAndCheckBatchReadWrite(
- true, false, "dt >= '2022-01-01'", Collections.emptyList(), dailyRates());
+ true, false, "dt >= '2022-01-01'", Collections.emptyList(), dailyRates().f0);
// test field filter
collectAndCheckBatchReadWrite(
@@ -254,7 +240,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
false,
"dt = '2022-01-01' OR rate > 115",
Collections.emptyList(),
- dailyRates());
+ dailyRates().f0);
// test projection
collectAndCheckBatchReadWrite(
@@ -292,7 +278,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
changelogRow("+I", "US Dollar", 102L),
changelogRow("+I", "Yen", 1L),
changelogRow("+I", "Euro", 119L)));
- checkFileStorePath(tEnv, managedTable);
+ checkFileStorePath(tEnv, managedTable, null);
// overwrite the whole table
prepareEnvAndOverwrite(
@@ -303,6 +289,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
tEnv,
managedTable,
Collections.emptyMap(),
+ null,
Collections.singletonList(changelogRow("+I", "Euro", 100L)));
// test field filter
@@ -345,7 +332,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
// input is rates()
String managedTable =
collectAndCheckBatchReadWrite(false, false, null, Collections.emptyList(), rates());
- checkFileStorePath(tEnv, managedTable);
+ checkFileStorePath(tEnv, managedTable, null);
// test field filter
collectAndCheckBatchReadWrite(
@@ -402,7 +389,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
// part = 2022-01-02
changelogRow("+I", "Euro", 119L, "2022-01-02")));
String managedTable = tuple.f0;
- checkFileStorePath(tEnv, managedTable);
+ checkFileStorePath(tEnv, managedTable, dailyRatesChangelogWithoutUB().f1);
BlockingIterator<Row, Row> streamIter = tuple.f1;
// test log store in hybrid mode accepts all filters
@@ -440,6 +427,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
batchTableEnv,
managedTable,
Collections.emptyMap(),
+ null,
Arrays.asList(
// part = 2022-01-01
changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
@@ -514,7 +502,8 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
// part = 2022-01-01
changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
// part = 2022-01-02
- changelogRow("+I", "Euro", 119L, "2022-01-02"))));
+ changelogRow("+I", "Euro", 119L, "2022-01-02"))),
+ dailyRatesChangelogWithoutUB().f1);
// test partition filter
collectAndCheckStreamingReadWriteWithClose(
@@ -582,7 +571,8 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
// part = 2022-01-01
changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
// part = 2022-01-02
- changelogRow("+I", "Euro", 115L, "2022-01-02"))));
+ changelogRow("+I", "Euro", 115L, "2022-01-02"))),
+ dailyRatesChangelogWithUB().f1);
// test partition filter
collectAndCheckStreamingReadWriteWithClose(
@@ -673,7 +663,8 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
Collections.emptyList(),
Arrays.asList(
changelogRow("+I", "US Dollar", 102L),
- changelogRow("+I", "Euro", 119L))));
+ changelogRow("+I", "Euro", 119L))),
+ null);
// test field filter
collectAndCheckStreamingReadWriteWithClose(
@@ -723,7 +714,8 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
changelogRow("+I", "US Dollar", 102L),
changelogRow("+I", "Euro", 119L),
changelogRow("+I", null, 100L),
- changelogRow("+I", "HK Dollar", null))));
+ changelogRow("+I", "HK Dollar", null))),
+ null);
// test field filter
collectAndCheckStreamingReadWriteWithClose(
@@ -1113,7 +1105,7 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
// part = 2022-01-02
changelogRow("+I", "Euro", 119L, "2022-01-02")));
- collectChangelogFromTimestampAndCheck(true, true, false, 0, dailyRates());
+ collectChangelogFromTimestampAndCheck(true, true, false, 0, dailyRates().f0);
// input is rates()
collectChangelogFromTimestampAndCheck(
@@ -1175,6 +1167,36 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
changelogRow("+I", "Euro", 119L, "2022-01-02")));
}
+ @Test
+ public void testSourceParallelism() throws Exception {
+ String managedTable =
+ createSourceAndManagedTable(
+ false,
+ false,
+ false,
+ Collections.emptyList(),
+ Collections.emptyList())
+ .f1;
+
+ // without hint
+ String query = buildSimpleSelectQuery(managedTable, Collections.emptyMap());
+ assertThat(sourceParallelism(query)).isEqualTo(env.getParallelism());
+
+ // with hint
+ query =
+ buildSimpleSelectQuery(
+ managedTable, Collections.singletonMap(SCAN_PARALLELISM.key(), "66"));
+ assertThat(sourceParallelism(query)).isEqualTo(66);
+ }
+
+ @Test
+ public void testSinkParallelism() {
+ testSinkParallelism(null, env.getParallelism());
+ testSinkParallelism(23, 23);
+ }
+
+ // ------------------------ Tools ----------------------------------
+
private String collectAndCheckBatchReadWrite(
boolean partitioned,
boolean hasPk,
@@ -1182,18 +1204,16 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
List<String> projection,
List<Row> expected)
throws Exception {
- return collectAndCheckUnderSameEnv(
- false,
- false,
- true,
- partitioned,
- hasPk,
- true,
- Collections.emptyMap(),
- filter,
- projection,
- expected)
- .f0;
+ return collectAndCheckBatchReadWrite(
+ partitioned ? Collections.singletonList("dt") : Collections.emptyList(),
+ hasPk
+ ? partitioned
+ ? Arrays.asList("currency", "dt")
+ : Collections.singletonList("currency")
+ : Collections.emptyList(),
+ filter,
+ projection,
+ expected);
}
private String collectAndCheckStreamingReadWriteWithClose(
@@ -1205,20 +1225,18 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
List<String> projection,
List<Row> expected)
throws Exception {
- Tuple2<String, BlockingIterator<Row, Row>> tuple =
- collectAndCheckUnderSameEnv(
- true,
- enableLogStore,
- false,
- partitioned,
- hasPk,
- true,
- readHints,
- filter,
- projection,
- expected);
- tuple.f1.close();
- return tuple.f0;
+ return collectAndCheckStreamingReadWriteWithClose(
+ enableLogStore,
+ partitioned ? Collections.singletonList("dt") : Collections.emptyList(),
+ hasPk
+ ? partitioned
+ ? Arrays.asList("currency", "dt")
+ : Collections.singletonList("currency")
+ : Collections.emptyList(),
+ readHints,
+ filter,
+ projection,
+ expected);
}
private Tuple2<String, BlockingIterator<Row, Row>>
@@ -1228,8 +1246,13 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
List<String> projection,
List<Row> expected)
throws Exception {
- return collectAndCheckUnderSameEnv(
- true, true, false, true, true, true, readHints, filter, projection, expected);
+ return collectAndCheckStreamingReadWriteWithoutClose(
+ Collections.singletonList("dt"),
+ Arrays.asList("currency", "dt"),
+ readHints,
+ filter,
+ projection,
+ expected);
}
private void collectLatestLogAndCheck(
@@ -1240,23 +1263,17 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
List<String> projection,
List<Row> expected)
throws Exception {
- Map<String, String> hints = new HashMap<>();
- hints.put(
- LOG_PREFIX + LogOptions.SCAN.key(),
- LogOptions.LogStartupMode.LATEST.name().toLowerCase());
- collectAndCheckUnderSameEnv(
- true,
- true,
- insertOnly,
- partitioned,
- hasPk,
- false,
- hints,
- filter,
- projection,
- expected)
- .f1
- .close();
+ collectLatestLogAndCheck(
+ insertOnly,
+ partitioned ? Collections.singletonList("dt") : Collections.emptyList(),
+ hasPk
+ ? partitioned
+ ? Arrays.asList("currency", "dt")
+ : Collections.singletonList("currency")
+ : Collections.emptyList(),
+ filter,
+ projection,
+ expected);
}
private void collectChangelogFromTimestampAndCheck(
@@ -1266,502 +1283,16 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
long timestamp,
List<Row> expected)
throws Exception {
- Map<String, String> hints = new HashMap<>();
- hints.put(LOG_PREFIX + LogOptions.SCAN.key(), "from-timestamp");
- hints.put(LOG_PREFIX + LogOptions.SCAN_TIMESTAMP_MILLS.key(), String.valueOf(timestamp));
- collectAndCheckUnderSameEnv(
- true,
- true,
- insertOnly,
- partitioned,
- hasPk,
- true,
- hints,
- null,
- Collections.emptyList(),
- expected)
- .f1
- .close();
- }
-
- private Tuple2<String, String> createSourceAndManagedTable(
- boolean streaming,
- boolean enableLogStore,
- boolean insertOnly,
- boolean partitioned,
- boolean hasPk)
- throws Exception {
- Map<String, String> tableOptions = new HashMap<>();
- rootPath = TEMPORARY_FOLDER.newFolder().getPath();
- tableOptions.put(FileStoreOptions.FILE_PATH.key(), rootPath);
- if (enableLogStore) {
- tableOptions.put(LOG_SYSTEM.key(), "kafka");
- tableOptions.put(LOG_PREFIX + BOOTSTRAP_SERVERS.key(), getBootstrapServers());
- }
- String sourceTable = "source_table_" + UUID.randomUUID();
- String managedTable = "managed_table_" + UUID.randomUUID();
- EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance().inStreamingMode();
- String helperTableDdl;
- if (streaming) {
- helperTableDdl =
- insertOnly
- ? prepareHelperSourceWithInsertOnlyRecords(
- sourceTable, partitioned, hasPk)
- : prepareHelperSourceWithChangelogRecords(
- sourceTable, partitioned, hasPk);
- env = buildStreamEnv();
- builder.inStreamingMode();
- } else {
- helperTableDdl =
- prepareHelperSourceWithInsertOnlyRecords(sourceTable, partitioned, hasPk);
- env = buildBatchEnv();
- builder.inBatchMode();
- }
- String managedTableDdl = prepareManagedTableDdl(sourceTable, managedTable, tableOptions);
-
- tEnv = StreamTableEnvironment.create(env, builder.build());
- tEnv.executeSql(helperTableDdl);
- tEnv.executeSql(managedTableDdl);
- return new Tuple2<>(sourceTable, managedTable);
- }
-
- private Tuple2<String, BlockingIterator<Row, Row>> collectAndCheckUnderSameEnv(
- boolean streaming,
- boolean enableLogStore,
- boolean insertOnly,
- boolean partitioned,
- boolean hasPk,
- boolean writeFirst,
- Map<String, String> readHints,
- @Nullable String filter,
- List<String> projection,
- List<Row> expected)
- throws Exception {
- Tuple2<String, String> tables =
- createSourceAndManagedTable(
- streaming, enableLogStore, insertOnly, partitioned, hasPk);
- String sourceTable = tables.f0;
- String managedTable = tables.f1;
-
- String insertQuery = prepareInsertIntoQuery(sourceTable, managedTable);
- String selectQuery = prepareSelectQuery(managedTable, readHints, filter, projection);
-
- BlockingIterator<Row, Row> iterator;
- if (writeFirst) {
- tEnv.executeSql(insertQuery).await();
- iterator = BlockingIterator.of(tEnv.executeSql(selectQuery).collect());
- } else {
- iterator = BlockingIterator.of(tEnv.executeSql(selectQuery).collect());
- tEnv.executeSql(insertQuery).await();
- }
- if (expected.isEmpty()) {
- assertNoMoreRecords(iterator);
- } else {
- assertThat(iterator.collect(expected.size(), 10, TimeUnit.SECONDS))
- .containsExactlyInAnyOrderElementsOf(expected);
- }
- return Tuple2.of(managedTable, iterator);
- }
-
- private void prepareEnvAndOverwrite(
- String managedTable,
- Map<String, String> staticPartitions,
- List<String[]> overwriteRecords)
- throws Exception {
- final StreamTableEnvironment batchEnv =
- StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
- registerTable(batchEnv, managedTable);
- String insertQuery =
- prepareInsertOverwriteQuery(managedTable, staticPartitions, overwriteRecords);
- batchEnv.executeSql(insertQuery).await();
- }
-
- private void registerTable(StreamTableEnvironment tEnvToRegister, String managedTable)
- throws Exception {
- String cat = this.tEnv.getCurrentCatalog();
- String db = this.tEnv.getCurrentDatabase();
- ObjectPath objectPath = new ObjectPath(db, managedTable);
- CatalogBaseTable table = this.tEnv.getCatalog(cat).get().getTable(objectPath);
- tEnvToRegister.getCatalog(cat).get().createTable(objectPath, table, false);
- }
-
- private BlockingIterator<Row, Row> collect(
- StreamTableEnvironment tEnv, String selectQuery, int expectedSize, List<Row> actual)
- throws Exception {
- TableResult result = tEnv.executeSql(selectQuery);
- BlockingIterator<Row, Row> iterator = BlockingIterator.of(result.collect());
- actual.addAll(iterator.collect(expectedSize));
- return iterator;
- }
-
- private BlockingIterator<Row, Row> collectAndCheck(
- StreamTableEnvironment tEnv,
- String managedTable,
- Map<String, String> hints,
- List<Row> expectedRecords)
- throws Exception {
- List<Row> actual = new ArrayList<>();
- BlockingIterator<Row, Row> iterator =
- collect(
- tEnv,
- prepareSimpleSelectQuery(managedTable, hints),
- expectedRecords.size(),
- actual);
- assertThat(actual).containsExactlyInAnyOrderElementsOf(expectedRecords);
- return iterator;
- }
-
- private void checkFileStorePath(StreamTableEnvironment tEnv, String managedTable) {
- String relativeFilePath =
- FileStoreOptions.relativeTablePath(
- ObjectIdentifier.of(
- tEnv.getCurrentCatalog(), tEnv.getCurrentDatabase(), managedTable));
- // check snapshot file path
- assertThat(Paths.get(rootPath, relativeFilePath, "snapshot")).exists();
- // check manifest file path
- assertThat(Paths.get(rootPath, relativeFilePath, "manifest")).exists();
- }
-
- private static void assertNoMoreRecords(BlockingIterator<Row, Row> iterator) {
- List<Row> expectedRecords = Collections.emptyList();
- try {
- // set expectation size to 1 to let time pass by until timeout
- // just wait 5s to avoid too long time
- expectedRecords = iterator.collect(1, 5L, TimeUnit.SECONDS);
- iterator.close();
- } catch (Exception ignored) {
- // don't throw exception
- }
- assertThat(expectedRecords).isEmpty();
- }
-
- private static String prepareManagedTableDdl(
- String sourceTableName, String managedTableName, Map<String, String> tableOptions) {
- StringBuilder ddl =
- new StringBuilder("CREATE TABLE IF NOT EXISTS ")
- .append(String.format("`%s`", managedTableName));
- ddl.append(prepareOptions(tableOptions))
- .append(String.format(" LIKE `%s` (EXCLUDING OPTIONS)\n", sourceTableName));
- return ddl.toString();
- }
-
- private static String prepareOptions(Map<String, String> tableOptions) {
- StringBuilder with = new StringBuilder();
- if (tableOptions.size() > 0) {
- with.append(" WITH (\n");
- tableOptions.forEach(
- (k, v) ->
- with.append(" ")
- .append(String.format("'%s'", k))
- .append(" = ")
- .append(String.format("'%s',\n", v)));
- int len = with.length();
- with.delete(len - 2, len);
- with.append(")");
- }
- return with.toString();
- }
-
- private static String prepareInsertIntoQuery(String sourceTableName, String managedTableName) {
- return prepareInsertIntoQuery(
- sourceTableName, managedTableName, Collections.emptyMap(), Collections.emptyMap());
- }
-
- private static String prepareInsertIntoQuery(
- String sourceTableName,
- String managedTableName,
- Map<String, String> partitions,
- Map<String, String> hints) {
- StringBuilder insertDmlBuilder =
- new StringBuilder(String.format("INSERT INTO `%s`", managedTableName));
- if (partitions.size() > 0) {
- insertDmlBuilder.append(" PARTITION (");
- partitions.forEach(
- (k, v) -> {
- insertDmlBuilder.append(String.format("'%s'", k));
- insertDmlBuilder.append(" = ");
- insertDmlBuilder.append(String.format("'%s', ", v));
- });
- int len = insertDmlBuilder.length();
- insertDmlBuilder.deleteCharAt(len - 1);
- insertDmlBuilder.append(")");
- }
- insertDmlBuilder.append(String.format("\n SELECT * FROM `%s`", sourceTableName));
- insertDmlBuilder.append(buildHints(hints));
-
- return insertDmlBuilder.toString();
- }
-
- private static String prepareInsertOverwriteQuery(
- String managedTableName,
- Map<String, String> staticPartitions,
- List<String[]> overwriteRecords) {
- StringBuilder insertDmlBuilder =
- new StringBuilder(String.format("INSERT OVERWRITE `%s`", managedTableName));
- if (staticPartitions.size() > 0) {
- insertDmlBuilder.append(" PARTITION (");
- staticPartitions.forEach(
- (k, v) -> {
- insertDmlBuilder.append(String.format("%s", k));
- insertDmlBuilder.append(" = ");
- insertDmlBuilder.append(String.format("%s, ", v));
- });
- int len = insertDmlBuilder.length();
- insertDmlBuilder.delete(len - 2, len);
- insertDmlBuilder.append(")");
- }
- insertDmlBuilder.append("\n VALUES ");
- overwriteRecords.forEach(
- record -> {
- int arity = record.length;
- insertDmlBuilder.append("(");
- IntStream.range(0, arity)
- .forEach(i -> insertDmlBuilder.append(record[i]).append(", "));
-
- if (arity > 0) {
- int len = insertDmlBuilder.length();
- insertDmlBuilder.delete(len - 2, len);
- }
- insertDmlBuilder.append("), ");
- });
- int len = insertDmlBuilder.length();
- insertDmlBuilder.delete(len - 2, len);
- return insertDmlBuilder.toString();
- }
-
- private static String prepareSimpleSelectQuery(String tableName, Map<String, String> hints) {
- return prepareSelectQuery(tableName, hints, null, Collections.emptyList());
- }
-
- private static String prepareSelectQuery(
- String tableName,
- Map<String, String> hints,
- @Nullable String filter,
- List<String> projections) {
- StringBuilder queryBuilder =
- new StringBuilder(
- String.format(
- "SELECT %s FROM `%s` %s",
- projections.isEmpty() ? "*" : String.join(", ", projections),
- tableName,
- buildHints(hints)));
- if (filter != null) {
- queryBuilder.append("\nWHERE ").append(filter);
- }
- return queryBuilder.toString();
- }
-
- private static String buildHints(Map<String, String> hints) {
- if (hints.size() > 0) {
- String hintString =
- hints.entrySet().stream()
- .map(
- entry ->
- String.format(
- "'%s' = '%s'",
- entry.getKey(), entry.getValue()))
- .collect(Collectors.joining(", "));
- return "/*+ OPTIONS (" + hintString + ") */";
- }
- return "";
- }
-
- private static String prepareHelperSourceWithInsertOnlyRecords(
- String sourceTable, boolean partitioned, boolean hasPk) {
- return prepareHelperSourceRecords(
- RuntimeExecutionMode.BATCH, sourceTable, partitioned, hasPk);
- }
-
- private static String prepareHelperSourceWithChangelogRecords(
- String sourceTable, boolean partitioned, boolean hasPk) {
- return prepareHelperSourceRecords(
- RuntimeExecutionMode.STREAMING, sourceTable, partitioned, hasPk);
- }
-
- /**
- * Prepare helper source table ddl according to different input parameter.
- *
- * <pre> E.g. pk with partition
- * {@code
- * CREATE TABLE source_table (
- * currency STRING,
- * rate BIGINT,
- * dt STRING) PARTITIONED BY (dt)
- * WITH (
- * 'connector' = 'values',
- * 'bounded' = executionMode == RuntimeExecutionMode.BATCH,
- * 'partition-list' = '...'
- * )
- * }
- * </pre>
- *
- * @param executionMode is used to calculate {@code bounded}
- * @param sourceTable source table name
- * @param partitioned is used to calculate {@code partition-list}
- * @param hasPk
- * @return helper source ddl
- */
- private static String prepareHelperSourceRecords(
- RuntimeExecutionMode executionMode,
- String sourceTable,
- boolean partitioned,
- boolean hasPk) {
- boolean bounded = executionMode == RuntimeExecutionMode.BATCH;
- String changelogMode = bounded ? "I" : hasPk ? "I,UA,D" : "I,UA,UB,D";
- StringBuilder ddlBuilder =
- new StringBuilder(String.format("CREATE TABLE `%s` (\n", sourceTable))
- .append(" currency STRING,\n")
- .append(" rate BIGINT");
- if (partitioned) {
- ddlBuilder.append(",\n dt STRING");
- }
- if (hasPk) {
- ddlBuilder.append(", \n PRIMARY KEY (currency");
- if (partitioned) {
- ddlBuilder.append(", dt");
- }
- ddlBuilder.append(") NOT ENFORCED\n");
- } else {
- ddlBuilder.append("\n");
- }
- ddlBuilder.append(")");
- if (partitioned) {
- ddlBuilder.append(" PARTITIONED BY (dt)\n");
- }
- List<Row> input;
- if (bounded) {
- input = partitioned ? dailyRates() : rates();
- } else {
- if (hasPk) {
- input = partitioned ? dailyRatesChangelogWithoutUB() : ratesChangelogWithoutUB();
- } else {
- input = partitioned ? dailyRatesChangelogWithUB() : ratesChangelogWithUB();
- }
- }
- ddlBuilder.append(
- String.format(
- " WITH (\n"
- + " 'connector' = 'values',\n"
- + " 'bounded' = '%s',\n"
- + " 'data-id' = '%s',\n"
- + " 'changelog-mode' = '%s',\n"
- + " 'disable-lookup' = 'true',\n"
- + " 'partition-list' = '%s'\n"
- + ")",
- bounded,
- registerData(input),
- changelogMode,
- partitioned ? "dt:2022-01-01;dt:2022-01-02" : ""));
- return ddlBuilder.toString();
- }
-
- private static List<Row> rates() {
- return Arrays.asList(
- changelogRow("+I", "US Dollar", 102L),
- changelogRow("+I", "Euro", 114L),
- changelogRow("+I", "Yen", 1L),
- changelogRow("+I", "Euro", 114L),
- changelogRow("+I", "Euro", 119L));
- }
-
- private static List<Row> dailyRates() {
- return Arrays.asList(
- // part = 2022-01-01
- changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
- changelogRow("+I", "Euro", 114L, "2022-01-01"),
- changelogRow("+I", "Yen", 1L, "2022-01-01"),
- changelogRow("+I", "Euro", 114L, "2022-01-01"),
- changelogRow("+I", "US Dollar", 114L, "2022-01-01"),
- // part = 2022-01-02
- changelogRow("+I", "Euro", 119L, "2022-01-02"));
- }
-
- static List<Row> ratesChangelogWithoutUB() {
- return Arrays.asList(
- changelogRow("+I", "US Dollar", 102L),
- changelogRow("+I", "Euro", 114L),
- changelogRow("+I", "Yen", 1L),
- changelogRow("+U", "Euro", 116L),
- changelogRow("-D", "Euro", 116L),
- changelogRow("+I", "Euro", 119L),
- changelogRow("+U", "Euro", 119L),
- changelogRow("-D", "Yen", 1L));
- }
-
- static List<Row> dailyRatesChangelogWithoutUB() {
- return Arrays.asList(
- // part = 2022-01-01
- changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
- changelogRow("+I", "Euro", 114L, "2022-01-01"),
- changelogRow("+I", "Yen", 1L, "2022-01-01"),
- changelogRow("+U", "Euro", 116L, "2022-01-01"),
- changelogRow("-D", "Yen", 1L, "2022-01-01"),
- changelogRow("-D", "Euro", 116L, "2022-01-01"),
- // part = 2022-01-02
- changelogRow("+I", "Euro", 119L, "2022-01-02"),
- changelogRow("+U", "Euro", 119L, "2022-01-02"));
- }
-
- private static List<Row> ratesChangelogWithUB() {
- return Arrays.asList(
- changelogRow("+I", "US Dollar", 102L),
- changelogRow("+I", "Euro", 114L),
- changelogRow("+I", "Yen", 1L),
- changelogRow("-U", "Euro", 114L),
- changelogRow("+U", "Euro", 116L),
- changelogRow("-D", "Euro", 116L),
- changelogRow("+I", "Euro", 119L),
- changelogRow("-U", "Euro", 119L),
- changelogRow("+U", "Euro", 119L),
- changelogRow("-D", "Yen", 1L),
- changelogRow("+I", null, 100L),
- changelogRow("+I", "HK Dollar", null));
- }
-
- private static List<Row> dailyRatesChangelogWithUB() {
- return Arrays.asList(
- // part = 2022-01-01
- changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
- changelogRow("+I", "Euro", 116L, "2022-01-01"),
- changelogRow("-D", "Euro", 116L, "2022-01-01"),
- changelogRow("+I", "Yen", 1L, "2022-01-01"),
- changelogRow("-D", "Yen", 1L, "2022-01-01"),
- // part = 2022-01-02
- changelogRow("+I", "Euro", 114L, "2022-01-02"),
- changelogRow("-U", "Euro", 114L, "2022-01-02"),
- changelogRow("+U", "Euro", 119L, "2022-01-02"),
- changelogRow("-D", "Euro", 119L, "2022-01-02"),
- changelogRow("+I", "Euro", 115L, "2022-01-02"));
- }
-
- private static StreamExecutionEnvironment buildStreamEnv() {
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRuntimeMode(RuntimeExecutionMode.STREAMING);
- env.enableCheckpointing(100);
- env.setParallelism(2);
- return env;
- }
-
- private static StreamExecutionEnvironment buildBatchEnv() {
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setRuntimeMode(RuntimeExecutionMode.BATCH);
- env.setParallelism(2);
- return env;
- }
-
- @Test
- public void testSourceParallelism() throws Exception {
- String managedTable = createSourceAndManagedTable(false, false, false, false, false).f1;
-
- // without hint
- String query = prepareSimpleSelectQuery(managedTable, Collections.emptyMap());
- assertThat(sourceParallelism(query)).isEqualTo(env.getParallelism());
-
- // with hint
- query =
- prepareSimpleSelectQuery(
- managedTable, Collections.singletonMap(SCAN_PARALLELISM.key(), "66"));
- assertThat(sourceParallelism(query)).isEqualTo(66);
+ collectChangelogFromTimestampAndCheck(
+ insertOnly,
+ partitioned ? Collections.singletonList("dt") : Collections.emptyList(),
+ hasPk
+ ? partitioned
+ ? Arrays.asList("currency", "dt")
+ : Collections.singletonList("currency")
+ : Collections.emptyList(),
+ timestamp,
+ expected);
}
private int sourceParallelism(String sql) {
@@ -1769,30 +1300,23 @@ public class ReadWriteTableITCase extends KafkaTableTestBase {
return stream.getParallelism();
}
- @Test
- public void testSinkParallelism() {
- testSinkParallelism(null, env.getParallelism());
- testSinkParallelism(23, 23);
- }
-
private void testSinkParallelism(Integer configParallelism, int expectedParallelism) {
// 1. create a mock table sink
- ResolvedSchema schema = ResolvedSchema.of(Column.physical("a", DataTypes.STRING()));
Map<String, String> options = new HashMap<>();
if (configParallelism != null) {
options.put(SINK_PARALLELISM.key(), configParallelism.toString());
}
- Context context =
+ DynamicTableFactory.Context context =
new FactoryUtil.DefaultDynamicTableContext(
ObjectIdentifier.of("default", "default", "t1"),
- new ResolvedCatalogTable(
- CatalogTable.of(
- Schema.newBuilder().fromResolvedSchema(schema).build(),
- "mock context",
- Collections.emptyList(),
- options),
- schema),
+ createResolvedTable(
+ options,
+ RowType.of(
+ new LogicalType[] {new VarCharType(Integer.MAX_VALUE)},
+ new String[] {"a"}),
+ Collections.emptyList(),
+ Collections.emptyList()),
Collections.emptyMap(),
new Configuration(),
Thread.currentThread().getContextClassLoader(),
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
new file mode 100644
index 0000000..650166a
--- /dev/null
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.connector;
+
+import org.apache.flink.api.common.RuntimeExecutionMode;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.EnvironmentSettings;
+import org.apache.flink.table.api.TableResult;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.store.file.FileStoreOptions;
+import org.apache.flink.table.store.file.utils.BlockingIterator;
+import org.apache.flink.table.store.kafka.KafkaTableTestBase;
+import org.apache.flink.table.store.log.LogOptions;
+import org.apache.flink.types.Row;
+
+import javax.annotation.Nullable;
+
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.prepareHelperSourceWithChangelogRecords;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.prepareHelperSourceWithInsertOnlyRecords;
+import static org.apache.flink.table.store.connector.ShowCreateUtil.buildInsertIntoQuery;
+import static org.apache.flink.table.store.connector.ShowCreateUtil.buildInsertOverwriteQuery;
+import static org.apache.flink.table.store.connector.ShowCreateUtil.buildSelectQuery;
+import static org.apache.flink.table.store.connector.ShowCreateUtil.buildSimpleSelectQuery;
+import static org.apache.flink.table.store.connector.ShowCreateUtil.createTableLikeDDL;
+import static org.apache.flink.table.store.connector.TableStoreFactoryOptions.LOG_SYSTEM;
+import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
+import static org.apache.flink.table.store.log.LogOptions.LOG_PREFIX;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Table store read write test base. */
+public class ReadWriteTableTestBase extends KafkaTableTestBase {
+ protected String rootPath;
+
+ // ------------------------ Tools ----------------------------------
+
+ protected void checkFileStorePath(
+ StreamTableEnvironment tEnv, String managedTable, @Nullable String partitionList) {
+ String relativeFilePath =
+ FileStoreOptions.relativeTablePath(
+ ObjectIdentifier.of(
+ tEnv.getCurrentCatalog(), tEnv.getCurrentDatabase(), managedTable));
+ // check snapshot file path
+ assertThat(Paths.get(rootPath, relativeFilePath, "snapshot")).exists();
+ // check manifest file path
+ assertThat(Paths.get(rootPath, relativeFilePath, "manifest")).exists();
+ // check sst file path
+ if (partitionList == null) {
+ // at least exists bucket-0
+ assertThat(Paths.get(rootPath, relativeFilePath, "bucket-0")).exists();
+ } else {
+ Arrays.stream(partitionList.split(";"))
+ .map(str -> str.replaceAll(":", "="))
+ .map(str -> str.replaceAll(",", "/"))
+ .map(str -> str.replaceAll("null", "__DEFAULT_PARTITION__"))
+ .collect(Collectors.toList())
+ .forEach(
+ partition -> {
+ assertThat(Paths.get(rootPath, relativeFilePath, partition))
+ .exists();
+ assertThat(
+ Paths.get(
+ rootPath,
+ relativeFilePath,
+ partition,
+ "bucket-0"))
+ .exists();
+ });
+ }
+ }
+
+ protected static BlockingIterator<Row, Row> collectAndCheck(
+ StreamTableEnvironment tEnv,
+ String managedTable,
+ Map<String, String> hints,
+ @Nullable String filter,
+ List<Row> expectedRecords)
+ throws Exception {
+ List<Row> actual = new ArrayList<>();
+ BlockingIterator<Row, Row> iterator =
+ collect(
+ tEnv,
+ filter == null
+ ? buildSimpleSelectQuery(managedTable, hints)
+ : buildSelectQuery(
+ managedTable, hints, filter, Collections.emptyList()),
+ expectedRecords.size(),
+ actual);
+ assertThat(actual).containsExactlyInAnyOrderElementsOf(expectedRecords);
+ return iterator;
+ }
+
+ protected static BlockingIterator<Row, Row> collect(
+ StreamTableEnvironment tEnv, String selectQuery, int expectedSize, List<Row> actual)
+ throws Exception {
+ TableResult result = tEnv.executeSql(selectQuery);
+ BlockingIterator<Row, Row> iterator = BlockingIterator.of(result.collect());
+ actual.addAll(iterator.collect(expectedSize));
+ return iterator;
+ }
+
+ protected static void assertNoMoreRecords(BlockingIterator<Row, Row> iterator)
+ throws Exception {
+ List<Row> expectedRecords = Collections.emptyList();
+ try {
+ // set expectation size to 1 to let time pass by until timeout
+ // just wait 5s to avoid too long time
+ expectedRecords = iterator.collect(1, 5L, TimeUnit.SECONDS);
+ iterator.close();
+ } catch (TimeoutException ignored) {
+ // don't throw exception
+ }
+ assertThat(expectedRecords).isEmpty();
+ }
+
+ protected String collectAndCheckBatchReadWrite(
+ List<String> partitions,
+ List<String> primaryKeys,
+ @Nullable String filter,
+ List<String> projection,
+ List<Row> expected)
+ throws Exception {
+ return collectAndCheckUnderSameEnv(
+ false,
+ false,
+ true,
+ partitions,
+ primaryKeys,
+ true,
+ Collections.emptyMap(),
+ filter,
+ projection,
+ expected)
+ .f0;
+ }
+
+ protected String collectAndCheckStreamingReadWriteWithClose(
+ boolean enableLogStore,
+ List<String> partitions,
+ List<String> primaryKeys,
+ Map<String, String> readHints,
+ @Nullable String filter,
+ List<String> projection,
+ List<Row> expected)
+ throws Exception {
+ Tuple2<String, BlockingIterator<Row, Row>> tuple =
+ collectAndCheckUnderSameEnv(
+ true,
+ enableLogStore,
+ false,
+ partitions,
+ primaryKeys,
+ true,
+ readHints,
+ filter,
+ projection,
+ expected);
+ tuple.f1.close();
+ return tuple.f0;
+ }
+
+ protected Tuple2<String, BlockingIterator<Row, Row>>
+ collectAndCheckStreamingReadWriteWithoutClose(
+ List<String> partitions,
+ List<String> primaryKeys,
+ Map<String, String> readHints,
+ @Nullable String filter,
+ List<String> projection,
+ List<Row> expected)
+ throws Exception {
+ return collectAndCheckUnderSameEnv(
+ true,
+ true,
+ false,
+ partitions,
+ primaryKeys,
+ true,
+ readHints,
+ filter,
+ projection,
+ expected);
+ }
+
+ protected void collectLatestLogAndCheck(
+ boolean insertOnly,
+ List<String> partitionKeys,
+ List<String> primaryKeys,
+ @Nullable String filter,
+ List<String> projection,
+ List<Row> expected)
+ throws Exception {
+ Map<String, String> hints = new HashMap<>();
+ hints.put(
+ LOG_PREFIX + LogOptions.SCAN.key(),
+ LogOptions.LogStartupMode.LATEST.name().toLowerCase());
+ collectAndCheckUnderSameEnv(
+ true,
+ true,
+ insertOnly,
+ partitionKeys,
+ primaryKeys,
+ false,
+ hints,
+ filter,
+ projection,
+ expected)
+ .f1
+ .close();
+ }
+
+ protected void collectChangelogFromTimestampAndCheck(
+ boolean insertOnly,
+ List<String> partitionKeys,
+ List<String> primaryKeys,
+ long timestamp,
+ List<Row> expected)
+ throws Exception {
+ Map<String, String> hints = new HashMap<>();
+ hints.put(LOG_PREFIX + LogOptions.SCAN.key(), "from-timestamp");
+ hints.put(LOG_PREFIX + LogOptions.SCAN_TIMESTAMP_MILLS.key(), String.valueOf(timestamp));
+ collectAndCheckUnderSameEnv(
+ true,
+ true,
+ insertOnly,
+ partitionKeys,
+ primaryKeys,
+ true,
+ hints,
+ null,
+ Collections.emptyList(),
+ expected)
+ .f1
+ .close();
+ }
+
+ protected Tuple2<String, String> createSourceAndManagedTable(
+ boolean streaming,
+ boolean enableLogStore,
+ boolean insertOnly,
+ List<String> partitionKeys,
+ List<String> primaryKeys)
+ throws Exception {
+ Map<String, String> tableOptions = new HashMap<>();
+ rootPath = TEMPORARY_FOLDER.newFolder().getPath();
+ tableOptions.put(FileStoreOptions.FILE_PATH.key(), rootPath);
+ if (enableLogStore) {
+ tableOptions.put(LOG_SYSTEM.key(), "kafka");
+ tableOptions.put(LOG_PREFIX + BOOTSTRAP_SERVERS.key(), getBootstrapServers());
+ }
+ String sourceTable = "source_table_" + UUID.randomUUID();
+ String managedTable = "managed_table_" + UUID.randomUUID();
+ EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance().inStreamingMode();
+ String helperTableDdl;
+ if (streaming) {
+ helperTableDdl =
+ insertOnly
+ ? prepareHelperSourceWithInsertOnlyRecords(
+ sourceTable, partitionKeys, primaryKeys)
+ : prepareHelperSourceWithChangelogRecords(
+ sourceTable, partitionKeys, primaryKeys);
+ env = buildStreamEnv();
+ builder.inStreamingMode();
+ } else {
+ helperTableDdl =
+ prepareHelperSourceWithInsertOnlyRecords(
+ sourceTable, partitionKeys, primaryKeys);
+ env = buildBatchEnv();
+ builder.inBatchMode();
+ }
+ String managedTableDdl = createTableLikeDDL(sourceTable, managedTable, tableOptions);
+
+ tEnv = StreamTableEnvironment.create(env, builder.build());
+ tEnv.executeSql(helperTableDdl);
+ tEnv.executeSql(managedTableDdl);
+ return new Tuple2<>(sourceTable, managedTable);
+ }
+
+ protected Tuple2<String, BlockingIterator<Row, Row>> collectAndCheckUnderSameEnv(
+ boolean streaming,
+ boolean enableLogStore,
+ boolean insertOnly,
+ List<String> partitionKeys,
+ List<String> primaryKeys,
+ boolean writeFirst,
+ Map<String, String> readHints,
+ @Nullable String filter,
+ List<String> projection,
+ List<Row> expected)
+ throws Exception {
+ Tuple2<String, String> tables =
+ createSourceAndManagedTable(
+ streaming, enableLogStore, insertOnly, partitionKeys, primaryKeys);
+ String sourceTable = tables.f0;
+ String managedTable = tables.f1;
+
+ String insertQuery = buildInsertIntoQuery(sourceTable, managedTable);
+ String selectQuery = buildSelectQuery(managedTable, readHints, filter, projection);
+
+ BlockingIterator<Row, Row> iterator;
+ if (writeFirst) {
+ tEnv.executeSql(insertQuery).await();
+ iterator = BlockingIterator.of(tEnv.executeSql(selectQuery).collect());
+ } else {
+ iterator = BlockingIterator.of(tEnv.executeSql(selectQuery).collect());
+ tEnv.executeSql(insertQuery).await();
+ }
+ if (expected.isEmpty()) {
+ assertNoMoreRecords(iterator);
+ } else {
+ assertThat(iterator.collect(expected.size(), 10, TimeUnit.SECONDS))
+ .containsExactlyInAnyOrderElementsOf(expected);
+ }
+ return Tuple2.of(managedTable, iterator);
+ }
+
+ protected void prepareEnvAndOverwrite(
+ String managedTable,
+ Map<String, String> staticPartitions,
+ List<String[]> overwriteRecords)
+ throws Exception {
+ prepareEnvAndOverwrite(
+ managedTable,
+ buildInsertOverwriteQuery(managedTable, staticPartitions, overwriteRecords));
+ }
+
+ protected void prepareEnvAndOverwrite(String managedTable, String query) throws Exception {
+ final StreamTableEnvironment batchEnv =
+ StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
+ registerTable(batchEnv, managedTable);
+ batchEnv.executeSql(query).await();
+ }
+
+ protected void registerTable(StreamTableEnvironment tEnvToRegister, String managedTable)
+ throws Exception {
+ String cat = tEnv.getCurrentCatalog();
+ String db = tEnv.getCurrentDatabase();
+ ObjectPath objectPath = new ObjectPath(db, managedTable);
+ CatalogBaseTable table = tEnv.getCatalog(cat).get().getTable(objectPath);
+ tEnvToRegister.getCatalog(cat).get().createTable(objectPath, table, false);
+ }
+
+ protected static StreamExecutionEnvironment buildStreamEnv() {
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setRuntimeMode(RuntimeExecutionMode.STREAMING);
+ env.enableCheckpointing(100);
+ env.setParallelism(2);
+ return env;
+ }
+
+ protected static StreamExecutionEnvironment buildBatchEnv() {
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setRuntimeMode(RuntimeExecutionMode.BATCH);
+ env.setParallelism(2);
+ return env;
+ }
+}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestUtil.java
new file mode 100644
index 0000000..6c86a8f
--- /dev/null
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestUtil.java
@@ -0,0 +1,534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.connector;
+
+import org.apache.flink.api.common.RuntimeExecutionMode;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.planner.factories.TestValuesTableFactory;
+import org.apache.flink.types.Row;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.flink.table.api.DataTypes.BIGINT;
+import static org.apache.flink.table.api.DataTypes.DOUBLE;
+import static org.apache.flink.table.api.DataTypes.STRING;
+import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
+import static org.apache.flink.table.planner.factories.TestValuesTableFactory.registerData;
+import static org.apache.flink.table.store.connector.ShowCreateUtil.buildShowCreateTable;
+import static org.apache.flink.table.store.connector.TableStoreTestBase.createResolvedTable;
+
+/** Data util for {@link ReadWriteTableITCase}. */
+public class ReadWriteTableTestUtil {
+
+ public static String prepareHelperSourceWithInsertOnlyRecords(
+ String sourceTable, List<String> partitionKeys, List<String> primaryKeys) {
+ return prepareHelperSourceRecords(
+ RuntimeExecutionMode.BATCH, sourceTable, partitionKeys, primaryKeys);
+ }
+
+ public static String prepareHelperSourceWithChangelogRecords(
+ String sourceTable, List<String> partitionKeys, List<String> primaryKeys) {
+ return prepareHelperSourceRecords(
+ RuntimeExecutionMode.STREAMING, sourceTable, partitionKeys, primaryKeys);
+ }
+
+ /**
+ * Prepare helper source table ddl according to different input parameter.
+ *
+ * <pre> E.g. pk with partition
+ * {@code
+ * CREATE TABLE source_table (
+ * currency STRING,
+ * rate BIGINT,
+ * dt STRING) PARTITIONED BY (dt)
+ * WITH (
+ * 'connector' = 'values',
+ * 'bounded' = executionMode == RuntimeExecutionMode.BATCH,
+ * 'partition-list' = '...'
+ * )
+ * }
+ * </pre>
+ *
+ * @param executionMode is used to calculate {@code bounded}
+ * @param sourceTable source table name
+ * @param partitionKeys is used to calculate {@code partition-list}
+ * @param primaryKeys
+ * @return helper source ddl
+ */
+ private static String prepareHelperSourceRecords(
+ RuntimeExecutionMode executionMode,
+ String sourceTable,
+ List<String> partitionKeys,
+ List<String> primaryKeys) {
+ Map<String, String> tableOptions = new HashMap<>();
+ boolean bounded = executionMode == RuntimeExecutionMode.BATCH;
+ String changelogMode = bounded ? "I" : primaryKeys.size() > 0 ? "I,UA,D" : "I,UA,UB,D";
+ List<String> exclusivePk = new ArrayList<>(primaryKeys);
+ exclusivePk.removeAll(partitionKeys);
+ Tuple2<List<Row>, String> tuple =
+ prepareHelperSource(bounded, exclusivePk.size(), partitionKeys.size());
+ String dataId = registerData(tuple.f0);
+ String partitionList = tuple.f1;
+ tableOptions.put("connector", TestValuesTableFactory.IDENTIFIER);
+ tableOptions.put("bounded", String.valueOf(bounded));
+ tableOptions.put("disable-lookup", String.valueOf(true));
+ tableOptions.put("changelog-mode", changelogMode);
+ tableOptions.put("data-id", dataId);
+ if (partitionList != null) {
+ tableOptions.put("partition-list", partitionList);
+ }
+ ResolvedCatalogTable table;
+ if (exclusivePk.size() > 1) {
+ table = prepareExchangeRateSource(partitionKeys, primaryKeys, tableOptions);
+ } else {
+ table = prepareRateSource(partitionKeys, primaryKeys, tableOptions);
+ }
+ return buildShowCreateTable(
+ table,
+ ObjectIdentifier.of("default_catalog", "default_database", sourceTable),
+ true);
+ }
+
+ private static Tuple2<List<Row>, String> prepareHelperSource(
+ boolean bounded, int pkSize, int partitionSize) {
+ List<Row> input;
+ String partitionList = null;
+ if (pkSize == 2) {
+ if (bounded) {
+ if (partitionSize == 0) {
+ // multi-pk, non-partitioned
+ input = exchangeRates();
+ } else if (partitionSize == 1) {
+ // multi-pk, single-partitioned
+ input = dailyExchangeRates().f0;
+ partitionList = dailyExchangeRates().f1;
+ } else {
+ // multi-pk, multi-partitioned
+ input = hourlyExchangeRates().f0;
+ partitionList = hourlyExchangeRates().f1;
+ }
+ } else {
+ if (partitionSize == 0) {
+ // multi-pk, non-partitioned
+ input = exchangeRatesChangelogWithoutUB();
+ } else if (partitionSize == 1) {
+ // multi-pk, single-partitioned
+ input = dailyExchangeRatesChangelogWithoutUB().f0;
+ partitionList = dailyExchangeRatesChangelogWithoutUB().f1;
+ } else {
+ // multi-pk, multi-partitioned
+ input = hourlyExchangeRatesChangelogWithoutUB().f0;
+ partitionList = hourlyExchangeRatesChangelogWithoutUB().f1;
+ }
+ }
+ } else if (pkSize == 1) {
+ if (bounded) {
+ if (partitionSize == 0) {
+ // single-pk, non-partitioned
+ input = rates();
+ } else if (partitionSize == 1) {
+ // single-pk, single-partitioned
+ input = dailyRates().f0;
+ partitionList = dailyRates().f1;
+ } else {
+ // single-pk, multi-partitioned
+ input = hourlyRates().f0;
+ partitionList = hourlyRates().f1;
+ }
+ } else {
+ if (partitionSize == 0) {
+ // single-pk, non-partitioned
+ input = ratesChangelogWithoutUB();
+ } else if (partitionSize == 1) {
+ // single-pk, single-partitioned
+ input = dailyRatesChangelogWithoutUB().f0;
+ partitionList = dailyRatesChangelogWithoutUB().f1;
+ } else {
+ // single-pk, multi-partitioned
+ input = hourlyRatesChangelogWithoutUB().f0;
+ partitionList = hourlyRatesChangelogWithoutUB().f1;
+ }
+ }
+ } else {
+ if (bounded) {
+ if (partitionSize == 0) {
+ // without pk, non-partitioned
+ input = rates();
+ } else if (partitionSize == 1) {
+ // without, single-partitioned
+ input = dailyRates().f0;
+ partitionList = dailyRates().f1;
+ } else {
+ // without pk, multi-partitioned
+ input = hourlyRates().f0;
+ partitionList = hourlyRates().f1;
+ }
+ } else {
+ if (partitionSize == 0) {
+ // without pk, non-partitioned
+ input = ratesChangelogWithUB();
+ } else if (partitionSize == 1) {
+ // without pk, single-partitioned
+ input = dailyRatesChangelogWithUB().f0;
+ partitionList = dailyRatesChangelogWithUB().f1;
+ } else {
+ // without pk, multi-partitioned
+ input = hourlyRatesChangelogWithUB().f0;
+ partitionList = hourlyRatesChangelogWithUB().f1;
+ }
+ }
+ }
+ return Tuple2.of(input, partitionList);
+ }
+
+ private static ResolvedCatalogTable prepareRateSource(
+ List<String> partitionKeys,
+ List<String> primaryKeys,
+ Map<String, String> tableOptions) {
+ List<Column> resolvedColumns = new ArrayList<>();
+ resolvedColumns.add(Column.physical("currency", STRING()));
+ resolvedColumns.add(Column.physical("rate", BIGINT()));
+ if (partitionKeys.size() > 0) {
+ resolvedColumns.add(Column.physical("dt", STRING()));
+ }
+ if (partitionKeys.size() == 2) {
+ resolvedColumns.add(Column.physical("hh", STRING()));
+ }
+ return createResolvedTable(tableOptions, resolvedColumns, partitionKeys, primaryKeys);
+ }
+
+ private static ResolvedCatalogTable prepareExchangeRateSource(
+ List<String> partitionKeys,
+ List<String> primaryKeys,
+ Map<String, String> tableOptions) {
+ List<Column> resolvedColumns = new ArrayList<>();
+ resolvedColumns.add(Column.physical("from_currency", STRING()));
+ resolvedColumns.add(Column.physical("to_currency", STRING()));
+ resolvedColumns.add(Column.physical("rate_by_to_currency", DOUBLE()));
+ if (partitionKeys.size() > 0) {
+ resolvedColumns.add(Column.physical("dt", STRING()));
+ }
+ if (partitionKeys.size() == 2) {
+ resolvedColumns.add(Column.physical("hh", STRING()));
+ }
+ return createResolvedTable(tableOptions, resolvedColumns, partitionKeys, primaryKeys);
+ }
+
+ public static List<Row> rates() {
+ // currency, rate
+ return Arrays.asList(
+ changelogRow("+I", "US Dollar", 102L),
+ changelogRow("+I", "Euro", 114L),
+ changelogRow("+I", "Yen", 1L),
+ changelogRow("+I", "Euro", 114L),
+ changelogRow("+I", "Euro", 119L));
+ }
+
+ public static List<Row> exchangeRates() {
+ // from_currency, to_currency, rate_by_to_currency
+ return Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d),
+ changelogRow("+I", "Singapore Dollar", "Yen", 122.46d));
+ }
+
+ public static Tuple2<List<Row>, String> dailyRates() {
+ // currency, rate, dt
+ return Tuple2.of(
+ Arrays.asList(
+ // dt = 2022-01-01
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01"),
+ // dt = 2022-01-02
+ changelogRow("+I", "Euro", 119L, "2022-01-02")),
+ "dt:2022-01-01;dt:2022-01-02");
+ }
+
+ public static Tuple2<List<Row>, String> dailyExchangeRates() {
+ // from_currency, to_currency, rate_by_to_currency, dt
+ return Tuple2.of(
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-02"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-01"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-01"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 122.46d, "2022-01-01")),
+ "dt:2022-01-01;dt:2022-01-02");
+ }
+
+ public static Tuple2<List<Row>, String> hourlyRates() {
+ // currency, rate, dt, hh
+ return Tuple2.of(
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 00
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "00"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "00"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "00"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "00"),
+ // dt = 2022-01-01, hh = 20
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "20"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "20"),
+ changelogRow("+I", "US Dollar", 114L, "2022-01-01", "20"),
+ // dt = 2022-01-02, hh = 12
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "12")),
+ "dt:2022-01-01,hh:00;dt:2022-01-01,hh:20;dt:2022-01-02,hh:12");
+ }
+
+ public static Tuple2<List<Row>, String> hourlyExchangeRates() {
+ // from_currency, to_currency, rate_by_to_currency, dt, hh
+ return Tuple2.of(
+ Arrays.asList(
+ // to_currency is USD, dt = 2022-01-01, hh = 11
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d, "2022-01-01", "11"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01", "11"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01", "11"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01", "11"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "11"),
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "11"),
+ // to_currency is USD, dt = 2022-01-01, hh = 12
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-01", "12"),
+ changelogRow("+I", "Euro", "US Dollar", 1.12d, "2022-01-01", "12"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.129d, "2022-01-01", "12"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01", "12"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.741d, "2022-01-01", "12"),
+ changelogRow("+I", "Yen", "US Dollar", 0.00812d, "2022-01-01", "12"),
+ // to_currency is Euro, dt = 2022-01-02, hh = 23
+ changelogRow("+I", "US Dollar", "Euro", 0.918d, "2022-01-02", "23"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "23")),
+ "dt:2022-01-01,hh:11;dt:2022-01-01,hh:12;dt:2022-01-02,hh:23");
+ }
+
+ public static List<Row> ratesChangelogWithoutUB() {
+ return Arrays.asList(
+ changelogRow("+I", "US Dollar", 102L),
+ changelogRow("+I", "Euro", 114L),
+ changelogRow("+I", "Yen", 1L),
+ changelogRow("+U", "Euro", 116L),
+ changelogRow("-D", "Euro", 116L),
+ changelogRow("+I", "Euro", 119L),
+ changelogRow("+U", "Euro", 119L),
+ changelogRow("-D", "Yen", 1L));
+ }
+
+ public static List<Row> exchangeRatesChangelogWithoutUB() {
+ // from_currency, to_currency, rate_by_to_currency
+ return Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d),
+ changelogRow("+I", "Euro", "US Dollar", 1.11d),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d),
+ changelogRow("+U", "Euro", "US Dollar", 1.12d),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d),
+ changelogRow("+U", "Yen", "US Dollar", 0.0081d),
+ changelogRow("-D", "US Dollar", "US Dollar", 1.0d),
+ changelogRow("-D", "Yen", "US Dollar", 0.0081d),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d),
+ changelogRow("+U", "Singapore Dollar", "Euro", 0.69d),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d),
+ changelogRow("-D", "Yen", "Yen", 1.0d),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122.46d),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122d));
+ }
+
+ public static Tuple2<List<Row>, String> dailyRatesChangelogWithoutUB() {
+ return Tuple2.of(
+ Arrays.asList(
+ // dt = 2022-01-01
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01"),
+ changelogRow("+U", "Euro", 116L, "2022-01-01"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01"),
+ // dt = 2022-01-02
+ changelogRow("+I", "Euro", 119L, "2022-01-02"),
+ changelogRow("+U", "Euro", 119L, "2022-01-02")),
+ "dt:2022-01-01;dt:2022-01-02");
+ }
+
+ public static Tuple2<List<Row>, String> dailyExchangeRatesChangelogWithoutUB() {
+ // from_currency, to_currency, rate_by_to_currency, dt
+ return Tuple2.of(
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", null, "2022-01-01"),
+ changelogRow("+I", "Euro", "US Dollar", null, "2022-01-01"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("-D", "Yen", "US Dollar", 0.0082d, "2022-01-01"),
+ changelogRow("+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-01"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0081d, "2022-01-01"),
+ changelogRow("+U", "Euro", "US Dollar", 1.11d, "2022-01-01"),
+ changelogRow("+U", "US Dollar", "US Dollar", 1.0d, "2022-01-01"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ changelogRow("-D", "Singapore Dollar", "Euro", 0.67d, "2022-01-02"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02"),
+ changelogRow("+U", "Singapore Dollar", "Yen", 122.46d, "2022-01-02")),
+ "dt:2022-01-01;dt:2022-01-02");
+ }
+
+ public static Tuple2<List<Row>, String> hourlyRatesChangelogWithoutUB() {
+ return Tuple2.of(
+ Arrays.asList(
+ // dt = 2022-01-01, hh = "15"
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 114L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("+U", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ // dt = 2022-01-02, hh = "23"
+ changelogRow("+I", "Euro", 119L, "2022-01-02", "23"),
+ changelogRow("+U", "Euro", 119L, "2022-01-02", "23")),
+ "dt:2022-01-01,hh:15;dt:2022-01-02,hh:23");
+ }
+
+ public static Tuple2<List<Row>, String> hourlyExchangeRatesChangelogWithoutUB() {
+ // from_currency, to_currency, rate_by_to_currency, dt, hh
+ return Tuple2.of(
+ Arrays.asList(
+ // to_currency is USD
+ changelogRow("+I", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", "US Dollar", null, "2022-01-02", "20"),
+ changelogRow("+I", "HK Dollar", "US Dollar", 0.13d, "2022-01-02", "20"),
+ changelogRow("+I", "Yen", "US Dollar", 0.0082d, "2022-01-02", "20"),
+ changelogRow(
+ "+I", "Singapore Dollar", "US Dollar", 0.74d, "2022-01-02", "20"),
+ changelogRow("+U", "Yen", "US Dollar", 0.0081d, "2022-01-02", "20"),
+ changelogRow("-D", "US Dollar", "US Dollar", 1.0d, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", "US Dollar", null, "2022-01-02", "20"),
+ changelogRow(
+ "+U", "Singapore Dollar", "US Dollar", 0.76d, "2022-01-02", "20"),
+ // to_currency is Euro
+ changelogRow("+I", "US Dollar", "Euro", 0.9d, "2022-01-02", "20"),
+ changelogRow("+I", "Singapore Dollar", "Euro", 0.67d, "2022-01-02", "20"),
+ changelogRow("+U", "Singapore Dollar", "Euro", null, "2022-01-02", "20"),
+ // to_currency is Yen
+ changelogRow("+I", "Yen", "Yen", 1.0d, "2022-01-02", "20"),
+ changelogRow("+I", "Chinese Yuan", "Yen", 19.25d, "2022-01-02", "20"),
+ changelogRow("+U", "Chinese Yuan", "Yen", 25.6d, "2022-01-02", "20"),
+ changelogRow("+I", "Singapore Dollar", "Yen", 90.32d, "2022-01-02", "20"),
+ changelogRow("+I", "US Dollar", "Yen", 122.46d, "2022-01-02", "21"),
+ changelogRow("+U", "Singapore Dollar", "Yen", 90.1d, "2022-01-02", "20")),
+ "dt:2022-01-02,hh:20;dt:2022-01-02,hh:21");
+ }
+
+ public static List<Row> ratesChangelogWithUB() {
+ return Arrays.asList(
+ changelogRow("+I", "US Dollar", 102L),
+ changelogRow("+I", "Euro", 114L),
+ changelogRow("+I", "Yen", 1L),
+ changelogRow("-U", "Euro", 114L),
+ changelogRow("+U", "Euro", 116L),
+ changelogRow("-D", "Euro", 116L),
+ changelogRow("+I", "Euro", 119L),
+ changelogRow("-U", "Euro", 119L),
+ changelogRow("+U", "Euro", 119L),
+ changelogRow("-D", "Yen", 1L),
+ changelogRow("+I", null, 100L),
+ changelogRow("+I", "HK Dollar", null));
+ }
+
+ public static Tuple2<List<Row>, String> dailyRatesChangelogWithUB() {
+ return Tuple2.of(
+ Arrays.asList(
+ // dt = 2022-01-01
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01"),
+ // dt = 2022-01-02
+ changelogRow("+I", "Euro", 114L, "2022-01-02"),
+ changelogRow("-U", "Euro", 114L, "2022-01-02"),
+ changelogRow("+U", "Euro", 119L, "2022-01-02"),
+ changelogRow("-D", "Euro", 119L, "2022-01-02"),
+ changelogRow("+I", "Euro", 115L, "2022-01-02")),
+ "dt:2022-01-01;dt:2022-01-02");
+ }
+
+ public static Tuple2<List<Row>, String> hourlyRatesChangelogWithUB() {
+ return Tuple2.of(
+ Arrays.asList(
+ // dt = 2022-01-01, hh = 15
+ changelogRow("+I", "US Dollar", 102L, "2022-01-01", "15"),
+ changelogRow("+I", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("-D", "Euro", 116L, "2022-01-01", "15"),
+ changelogRow("+I", "Yen", 1L, "2022-01-01", "15"),
+ changelogRow("-D", "Yen", 1L, "2022-01-01", "15"),
+ // dt = 2022-01-02, hh = 20
+ changelogRow("+I", "Euro", 114L, "2022-01-02", "20"),
+ changelogRow("-U", "Euro", 114L, "2022-01-02", "20"),
+ changelogRow("+U", "Euro", 119L, "2022-01-02", "20"),
+ changelogRow("-D", "Euro", 119L, "2022-01-02", "20"),
+ changelogRow("+I", "Euro", 115L, "2022-01-02", "20")),
+ "dt:2022-01-01,hh:15;dt:2022-01-02,hh:20");
+ }
+}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
index 118d5c3..a1fa137 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
@@ -29,9 +29,15 @@ import org.apache.flink.table.utils.EncodingUtils;
import org.apache.commons.lang3.StringUtils;
+import javax.annotation.Nullable;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
/**
* SHOW CREATE statement Util.
@@ -42,7 +48,104 @@ public class ShowCreateUtil {
private ShowCreateUtil() {}
- static String buildShowCreateTable(
+ public static String createTableLikeDDL(
+ String sourceTableName, String managedTableName, Map<String, String> tableOptions) {
+ StringBuilder ddl =
+ new StringBuilder("CREATE TABLE IF NOT EXISTS ")
+ .append(String.format("`%s`", managedTableName));
+ ddl.append(optionsToSql(tableOptions))
+ .append(String.format(" LIKE `%s` (EXCLUDING OPTIONS)\n", sourceTableName));
+ return ddl.toString();
+ }
+
+ public static String buildInsertOverwriteQuery(
+ String managedTableName,
+ Map<String, String> staticPartitions,
+ List<String[]> overwriteRecords) {
+ StringBuilder insertDmlBuilder =
+ new StringBuilder(String.format("INSERT OVERWRITE `%s`", managedTableName));
+ if (staticPartitions.size() > 0) {
+ String partitionString =
+ staticPartitions.entrySet().stream()
+ .map(
+ entry ->
+ String.format(
+ "%s = %s", entry.getKey(), entry.getValue()))
+ .collect(Collectors.joining(", "));
+ insertDmlBuilder.append(String.format("PARTITION (%s)", partitionString));
+ }
+ insertDmlBuilder.append("\n VALUES ");
+ overwriteRecords.forEach(
+ record -> {
+ int arity = record.length;
+ insertDmlBuilder.append("(");
+ IntStream.range(0, arity)
+ .forEach(i -> insertDmlBuilder.append(record[i]).append(", "));
+
+ if (arity > 0) {
+ int len = insertDmlBuilder.length();
+ insertDmlBuilder.delete(len - 2, len);
+ }
+ insertDmlBuilder.append("), ");
+ });
+ int len = insertDmlBuilder.length();
+ insertDmlBuilder.delete(len - 2, len);
+ return insertDmlBuilder.toString();
+ }
+
+ public static String buildInsertIntoQuery(String sourceTableName, String managedTableName) {
+ return buildInsertIntoQuery(
+ sourceTableName, managedTableName, Collections.emptyMap(), Collections.emptyMap());
+ }
+
+ public static String buildInsertIntoQuery(
+ String sourceTableName,
+ String managedTableName,
+ Map<String, String> partitions,
+ Map<String, String> hints) {
+ StringBuilder insertDmlBuilder =
+ new StringBuilder(String.format("INSERT INTO `%s`", managedTableName));
+ if (partitions.size() > 0) {
+ insertDmlBuilder.append(" PARTITION (");
+ partitions.forEach(
+ (k, v) -> {
+ insertDmlBuilder.append(String.format("'%s'", k));
+ insertDmlBuilder.append(" = ");
+ insertDmlBuilder.append(String.format("'%s', ", v));
+ });
+ int len = insertDmlBuilder.length();
+ insertDmlBuilder.deleteCharAt(len - 1);
+ insertDmlBuilder.append(")");
+ }
+ insertDmlBuilder.append(String.format("\n SELECT * FROM `%s`", sourceTableName));
+ insertDmlBuilder.append(buildHints(hints));
+
+ return insertDmlBuilder.toString();
+ }
+
+ public static String buildSimpleSelectQuery(String tableName, Map<String, String> hints) {
+ return buildSelectQuery(tableName, hints, null, Collections.emptyList());
+ }
+
+ public static String buildSelectQuery(
+ String tableName,
+ Map<String, String> hints,
+ @Nullable String filter,
+ List<String> projections) {
+ StringBuilder queryBuilder =
+ new StringBuilder(
+ String.format(
+ "SELECT %s FROM `%s` %s",
+ projections.isEmpty() ? "*" : String.join(", ", projections),
+ tableName,
+ buildHints(hints)));
+ if (filter != null) {
+ queryBuilder.append("\nWHERE ").append(filter);
+ }
+ return queryBuilder.toString();
+ }
+
+ public static String buildShowCreateTable(
ResolvedCatalogBaseTable<?> table,
ObjectIdentifier tableIdentifier,
boolean ignoreIfExists) {
@@ -75,13 +178,14 @@ public class ShowCreateUtil {
return sb.toString();
}
- static String extractFormattedColumns(ResolvedCatalogBaseTable<?> table, String printIndent) {
+ private static String extractFormattedColumns(
+ ResolvedCatalogBaseTable<?> table, String printIndent) {
return table.getResolvedSchema().getColumns().stream()
.map(column -> String.format("%s%s", printIndent, getColumnString(column)))
.collect(Collectors.joining(",\n"));
}
- static Optional<String> extractFormattedWatermarkSpecs(
+ private static Optional<String> extractFormattedWatermarkSpecs(
ResolvedCatalogBaseTable<?> table, String printIndent) {
if (table.getResolvedSchema().getWatermarkSpecs().isEmpty()) {
return Optional.empty();
@@ -101,7 +205,7 @@ public class ShowCreateUtil {
.collect(Collectors.joining("\n")));
}
- static Optional<String> extractFormattedComment(ResolvedCatalogBaseTable<?> table) {
+ private static Optional<String> extractFormattedComment(ResolvedCatalogBaseTable<?> table) {
String comment = table.getComment();
if (StringUtils.isNotEmpty(comment)) {
return Optional.of(EncodingUtils.escapeSingleQuotes(comment));
@@ -109,7 +213,8 @@ public class ShowCreateUtil {
return Optional.empty();
}
- static Optional<String> extractFormattedPartitionedInfo(ResolvedCatalogTable catalogTable) {
+ private static Optional<String> extractFormattedPartitionedInfo(
+ ResolvedCatalogTable catalogTable) {
if (!catalogTable.isPartitioned()) {
return Optional.empty();
}
@@ -119,7 +224,7 @@ public class ShowCreateUtil {
.collect(Collectors.joining(", ")));
}
- static Optional<String> extractFormattedOptions(
+ private static Optional<String> extractFormattedOptions(
ResolvedCatalogBaseTable<?> table, String printIndent) {
if (Objects.isNull(table.getOptions()) || table.getOptions().isEmpty()) {
return Optional.empty();
@@ -136,7 +241,8 @@ public class ShowCreateUtil {
.collect(Collectors.joining(",\n")));
}
- static String buildCreateFormattedPrefix(boolean ignoreIfExists, ObjectIdentifier identifier) {
+ private static String buildCreateFormattedPrefix(
+ boolean ignoreIfExists, ObjectIdentifier identifier) {
return String.format(
"CREATE TABLE%s %s (%s",
ignoreIfExists ? " IF NOT EXISTS " : "",
@@ -144,14 +250,14 @@ public class ShowCreateUtil {
System.lineSeparator());
}
- static Optional<String> extractFormattedPrimaryKey(
+ private static Optional<String> extractFormattedPrimaryKey(
ResolvedCatalogBaseTable<?> table, String printIndent) {
Optional<UniqueConstraint> primaryKey = table.getResolvedSchema().getPrimaryKey();
return primaryKey.map(
uniqueConstraint -> String.format("%s%s", printIndent, uniqueConstraint));
}
- static String getColumnString(Column column) {
+ private static String getColumnString(Column column) {
final StringBuilder sb = new StringBuilder();
sb.append(EncodingUtils.escapeIdentifier(column.getName()));
sb.append(" ");
@@ -177,4 +283,33 @@ public class ShowCreateUtil {
// TODO: Print the column comment until FLINK-18958 is fixed
return sb.toString();
}
+
+ private static String optionsToSql(Map<String, String> tableOptions) {
+ String optionsStr =
+ tableOptions.entrySet().stream()
+ .map(
+ entry ->
+ String.format(
+ "'%s' = '%s'", entry.getKey(), entry.getValue()))
+ .collect(Collectors.joining(",\n"));
+ if (StringUtils.isNotEmpty(optionsStr)) {
+ return "WITH (\n " + optionsStr + ")";
+ }
+ return optionsStr;
+ }
+
+ private static String buildHints(Map<String, String> hints) {
+ if (hints.size() > 0) {
+ String hintString =
+ hints.entrySet().stream()
+ .map(
+ entry ->
+ String.format(
+ "'%s' = '%s'",
+ entry.getKey(), entry.getValue()))
+ .collect(Collectors.joining(", "));
+ return "/*+ OPTIONS (" + hintString + ") */";
+ }
+ return "";
+ }
}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreFactoryTest.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreFactoryTest.java
index bfde9aa..4385050 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreFactoryTest.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreFactoryTest.java
@@ -163,10 +163,10 @@ public class TableStoreFactoryTest {
public void testBuildTableStore(
RowType rowType,
List<String> partitions,
- int[] pkIndex,
+ List<String> primaryKeys,
TableStoreTestBase.ExpectedResult expectedResult) {
ResolvedCatalogTable catalogTable =
- createResolvedTable(Collections.emptyMap(), rowType, partitions, pkIndex);
+ createResolvedTable(Collections.emptyMap(), rowType, partitions, primaryKeys);
context =
new FactoryUtil.DefaultDynamicTableContext(
TABLE_IDENTIFIER,
@@ -278,7 +278,7 @@ public class TableStoreFactoryTest {
Arguments.of(
rowType,
Arrays.asList("dt", "hr"),
- new int[] {0, 1, 2}, // pk is [dt, hr, shopId]
+ Arrays.asList("dt", "hr", "shopId"), // pk is [dt, hr, shopId]
new TableStoreTestBase.ExpectedResult().success(true));
// failed case: pk doesn't contain partition key
@@ -286,7 +286,7 @@ public class TableStoreFactoryTest {
Arguments.of(
rowType,
Arrays.asList("dt", "hr"),
- new int[] {2}, // pk is [shopId]
+ Collections.singletonList("shopId"), // pk is [shopId]
new TableStoreTestBase.ExpectedResult()
.success(false)
.expectedType(IllegalStateException.class)
@@ -298,7 +298,7 @@ public class TableStoreFactoryTest {
Arguments.of(
rowType,
Arrays.asList("dt", "hr", "shopId"),
- new int[] {0, 1, 2}, // pk is [dt, hr, shopId]
+ Arrays.asList("dt", "hr", "shopId"), // pk is [dt, hr, shopId]
new TableStoreTestBase.ExpectedResult()
.success(false)
.expectedType(IllegalStateException.class)
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreTestBase.java
index 191a197..1bba583 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreTestBase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreTestBase.java
@@ -44,7 +44,6 @@ import org.junit.Before;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Paths;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -119,30 +118,38 @@ public abstract class TableStoreTestBase extends KafkaTableTestBase {
}
protected static ResolvedCatalogTable createResolvedTable(
- Map<String, String> options, RowType rowType, List<String> partitionKeys, int[] pk) {
+ Map<String, String> options,
+ RowType rowType,
+ List<String> partitionKeys,
+ List<String> primaryKeys) {
List<String> fieldNames = rowType.getFieldNames();
List<DataType> fieldDataTypes =
rowType.getChildren().stream()
.map(TypeConversions::fromLogicalToDataType)
.collect(Collectors.toList());
+ List<Column> resolvedColumns =
+ IntStream.range(0, fieldNames.size())
+ .mapToObj(i -> Column.physical(fieldNames.get(i), fieldDataTypes.get(i)))
+ .collect(Collectors.toList());
+ return createResolvedTable(options, resolvedColumns, partitionKeys, primaryKeys);
+ }
+
+ protected static ResolvedCatalogTable createResolvedTable(
+ Map<String, String> options,
+ List<Column> resolvedColumns,
+ List<String> partitionKeys,
+ List<String> primaryKeys) {
+ UniqueConstraint constraint =
+ primaryKeys.isEmpty() ? null : UniqueConstraint.primaryKey("pk", primaryKeys);
+ ResolvedSchema resolvedSchema =
+ new ResolvedSchema(resolvedColumns, Collections.emptyList(), constraint);
CatalogTable origin =
CatalogTable.of(
- Schema.newBuilder().fromFields(fieldNames, fieldDataTypes).build(),
+ Schema.newBuilder().fromResolvedSchema(resolvedSchema).build(),
"a comment",
partitionKeys,
options);
- List<Column> resolvedColumns =
- IntStream.range(0, fieldNames.size())
- .mapToObj(i -> Column.physical(fieldNames.get(i), fieldDataTypes.get(i)))
- .collect(Collectors.toList());
- UniqueConstraint constraint = null;
- if (pk.length > 0) {
- List<String> pkNames =
- Arrays.stream(pk).mapToObj(fieldNames::get).collect(Collectors.toList());
- constraint = UniqueConstraint.primaryKey("pk", pkNames);
- }
- return new ResolvedCatalogTable(
- origin, new ResolvedSchema(resolvedColumns, Collections.emptyList(), constraint));
+ return new ResolvedCatalogTable(origin, resolvedSchema);
}
protected void deleteTablePath() {
[flink-table-store] 06/15: [hotfix] Value field should be named in value count mode
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 77192d78094429929a5d392f2d86a4a949df2d64
Author: Jane Chan <55...@users.noreply.github.com>
AuthorDate: Fri Apr 8 15:25:05 2022 +0800
[hotfix] Value field should be named in value count mode
This closes #79
---
.../flink/table/store/connector/TableStore.java | 5 +++-
.../store/connector/ReadWriteTableITCase.java | 27 ++++++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStore.java b/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStore.java
index 88826f0..37315c4 100644
--- a/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStore.java
+++ b/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStore.java
@@ -171,7 +171,10 @@ public class TableStore {
MergeFunction mergeFunction;
if (primaryKeys.length == 0) {
keyType = type;
- valueType = RowType.of(new BigIntType(false));
+ valueType =
+ RowType.of(
+ new LogicalType[] {new BigIntType(false)},
+ new String[] {"_VALUE_COUNT"});
mergeFunction = new ValueCountMergeFunction();
} else {
List<RowType.RowField> fields = TypeUtils.project(type, primaryKeys).getFields();
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
index 4596dec..bff091c 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
@@ -53,6 +53,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
+import static org.apache.flink.table.planner.factories.TestValuesTableFactory.registerData;
import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRates;
import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithUB;
import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.dailyRatesChangelogWithoutUB;
@@ -1197,6 +1198,32 @@ public class ReadWriteTableITCase extends ReadWriteTableTestBase {
testSinkParallelism(23, 23);
}
+ @Test
+ public void testQueryContainsDefaultFieldName() throws Exception {
+ rootPath = TEMPORARY_FOLDER.newFolder().getPath();
+ tEnv = StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
+ String id = registerData(Collections.singletonList(changelogRow("+I", 1, "abc")));
+ tEnv.executeSql(
+ String.format(
+ "create table dummy_source ("
+ + "f0 int, "
+ + "f1 string) with ("
+ + "'connector' = 'values', "
+ + "'bounded' = 'true', "
+ + "'data-id' = '%s')",
+ id));
+ tEnv.executeSql(
+ String.format(
+ "create table managed_table with ('file.path' = '%s') "
+ + "like dummy_source (excluding options)",
+ rootPath));
+ tEnv.executeSql("insert into managed_table select * from dummy_source").await();
+ BlockingIterator<Row, Row> iterator =
+ BlockingIterator.of(tEnv.executeSql("select * from managed_table").collect());
+ assertThat(iterator.collect(1, 5, TimeUnit.SECONDS))
+ .containsOnly(changelogRow("+I", 1, "abc"));
+ }
+
// ------------------------ Tools ----------------------------------
private String collectAndCheckBatchReadWrite(
[flink-table-store] 07/15: [hotfix] Fix junit4 dependency version
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit f8c1351ee777c07f066fcf8e8ee5e2f478529e92
Author: tsreaper <ts...@gmail.com>
AuthorDate: Fri Apr 8 15:30:29 2022 +0800
[hotfix] Fix junit4 dependency version
This closes #78
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 4dbcc41..7e8a601 100644
--- a/pom.xml
+++ b/pom.xml
@@ -167,7 +167,7 @@ under the License.
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>${junit.version}</version>
+ <version>${junit4.version}</version>
</dependency>
<dependency>
[flink-table-store] 13/15: [FLINK-27157] Add version field to manifest entries
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 88fcc837a6464b1b23462f1812e151aa01932383
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Apr 11 10:55:12 2022 +0800
[FLINK-27157] Add version field to manifest entries
This closes #85
---
.../file/manifest/ManifestEntrySerializer.java | 16 ++++--
.../table/store/file/manifest/ManifestFile.java | 5 +-
.../file/manifest/ManifestFileMetaSerializer.java | 16 ++++--
.../table/store/file/manifest/ManifestList.java | 4 +-
.../file/utils/VersionedObjectSerializer.java | 66 ++++++++++++++++++++++
5 files changed, 97 insertions(+), 10 deletions(-)
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestEntrySerializer.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestEntrySerializer.java
index 51736f6..4cce368 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestEntrySerializer.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestEntrySerializer.java
@@ -23,11 +23,11 @@ import org.apache.flink.table.data.RowData;
import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
import org.apache.flink.table.store.file.ValueKind;
import org.apache.flink.table.store.file.mergetree.sst.SstFileMetaSerializer;
-import org.apache.flink.table.store.file.utils.ObjectSerializer;
+import org.apache.flink.table.store.file.utils.VersionedObjectSerializer;
import org.apache.flink.table.types.logical.RowType;
/** Serializer for {@link ManifestEntry}. */
-public class ManifestEntrySerializer extends ObjectSerializer<ManifestEntry> {
+public class ManifestEntrySerializer extends VersionedObjectSerializer<ManifestEntry> {
private static final long serialVersionUID = 1L;
@@ -41,7 +41,12 @@ public class ManifestEntrySerializer extends ObjectSerializer<ManifestEntry> {
}
@Override
- public RowData toRow(ManifestEntry entry) {
+ public int getVersion() {
+ return 1;
+ }
+
+ @Override
+ public RowData convertTo(ManifestEntry entry) {
GenericRowData row = new GenericRowData(5);
row.setField(0, entry.kind().toByteValue());
row.setField(1, entry.partition());
@@ -52,7 +57,10 @@ public class ManifestEntrySerializer extends ObjectSerializer<ManifestEntry> {
}
@Override
- public ManifestEntry fromRow(RowData row) {
+ public ManifestEntry convertFrom(int version, RowData row) {
+ if (version != 1) {
+ throw new IllegalArgumentException("Unsupported version: " + version);
+ }
return new ManifestEntry(
ValueKind.fromByteValue(row.getByte(0)),
partitionSerializer
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFile.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFile.java
index 0fa2132..55d60ea 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFile.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFile.java
@@ -30,6 +30,7 @@ import org.apache.flink.table.store.file.stats.FieldStatsCollector;
import org.apache.flink.table.store.file.utils.FileStorePathFactory;
import org.apache.flink.table.store.file.utils.FileUtils;
import org.apache.flink.table.store.file.utils.RollingFile;
+import org.apache.flink.table.store.file.utils.VersionedObjectSerializer;
import org.apache.flink.table.types.logical.RowType;
import org.slf4j.Logger;
@@ -193,7 +194,9 @@ public class ManifestFile {
}
public ManifestFile create() {
- RowType entryType = ManifestEntry.schema(partitionType, keyType, valueType);
+ RowType entryType =
+ VersionedObjectSerializer.versionType(
+ ManifestEntry.schema(partitionType, keyType, valueType));
return new ManifestFile(
partitionType,
new ManifestEntrySerializer(partitionType, keyType, valueType),
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFileMetaSerializer.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFileMetaSerializer.java
index f0d2dd0..9ad3e6a 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFileMetaSerializer.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFileMetaSerializer.java
@@ -22,11 +22,11 @@ import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.StringData;
import org.apache.flink.table.store.file.stats.FieldStatsArraySerializer;
-import org.apache.flink.table.store.file.utils.ObjectSerializer;
+import org.apache.flink.table.store.file.utils.VersionedObjectSerializer;
import org.apache.flink.table.types.logical.RowType;
/** Serializer for {@link ManifestFileMeta}. */
-public class ManifestFileMetaSerializer extends ObjectSerializer<ManifestFileMeta> {
+public class ManifestFileMetaSerializer extends VersionedObjectSerializer<ManifestFileMeta> {
private static final long serialVersionUID = 1L;
@@ -38,7 +38,12 @@ public class ManifestFileMetaSerializer extends ObjectSerializer<ManifestFileMet
}
@Override
- public RowData toRow(ManifestFileMeta meta) {
+ public int getVersion() {
+ return 1;
+ }
+
+ @Override
+ public RowData convertTo(ManifestFileMeta meta) {
GenericRowData row = new GenericRowData(5);
row.setField(0, StringData.fromString(meta.fileName()));
row.setField(1, meta.fileSize());
@@ -49,7 +54,10 @@ public class ManifestFileMetaSerializer extends ObjectSerializer<ManifestFileMet
}
@Override
- public ManifestFileMeta fromRow(RowData row) {
+ public ManifestFileMeta convertFrom(int version, RowData row) {
+ if (version != 1) {
+ throw new IllegalArgumentException("Unsupported version: " + version);
+ }
return new ManifestFileMeta(
row.getString(0).toString(),
row.getLong(1),
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestList.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestList.java
index 82b64c0..1d0e78a 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestList.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestList.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.data.RowData;
import org.apache.flink.table.store.file.format.FileFormat;
import org.apache.flink.table.store.file.utils.FileStorePathFactory;
import org.apache.flink.table.store.file.utils.FileUtils;
+import org.apache.flink.table.store.file.utils.VersionedObjectSerializer;
import org.apache.flink.table.types.logical.RowType;
import java.io.IOException;
@@ -114,7 +115,8 @@ public class ManifestList {
}
public ManifestList create() {
- RowType metaType = ManifestFileMeta.schema(partitionType);
+ RowType metaType =
+ VersionedObjectSerializer.versionType(ManifestFileMeta.schema(partitionType));
return new ManifestList(
new ManifestFileMetaSerializer(partitionType),
fileFormat.createReaderFactory(metaType),
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/utils/VersionedObjectSerializer.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/utils/VersionedObjectSerializer.java
new file mode 100644
index 0000000..a07c474
--- /dev/null
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/utils/VersionedObjectSerializer.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.utils;
+
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.RowType;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** A {@link ObjectSerializer} for versioned serialization. */
+public abstract class VersionedObjectSerializer<T> extends ObjectSerializer<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ public VersionedObjectSerializer(RowType rowType) {
+ super(versionType(rowType));
+ }
+
+ public static RowType versionType(RowType rowType) {
+ List<RowType.RowField> fields = new ArrayList<>();
+ fields.add(new RowType.RowField("_VERSION", new IntType(false)));
+ fields.addAll(rowType.getFields());
+ return new RowType(fields);
+ }
+
+ /**
+ * Gets the version with which this serializer serializes.
+ *
+ * @return The version of the serialization schema.
+ */
+ public abstract int getVersion();
+
+ public abstract RowData convertTo(T record);
+
+ public abstract T convertFrom(int version, RowData row);
+
+ @Override
+ public final RowData toRow(T record) {
+ return new JoinedRowData().replace(GenericRowData.of(getVersion()), convertTo(record));
+ }
+
+ @Override
+ public final T fromRow(RowData row) {
+ return convertFrom(row.getInt(0), new OffsetRowData(row.getArity() - 1, 1).replace(row));
+ }
+}
[flink-table-store] 01/15: [FLINK-27070] Reuse FileFormat instead of DecodingFormat/EncodingFormat to ensure thread safety
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 69ecd0dd01bd6790e1b01f176471020034250229
Author: tsreaper <ts...@gmail.com>
AuthorDate: Wed Apr 6 16:06:30 2022 +0800
[FLINK-27070] Reuse FileFormat instead of DecodingFormat/EncodingFormat to ensure thread safety
This closes #74
---
.../flink/table/store/file/format/FileFormat.java | 6 +++++-
.../table/store/file/format/FileFormatImpl.java | 20 ++++++++++----------
.../table/store/file/manifest/ManifestFile.java | 12 +++++-------
.../table/store/file/manifest/ManifestList.java | 12 +++++-------
.../store/file/mergetree/sst/SstFileReader.java | 11 ++++-------
.../store/file/mergetree/sst/SstFileWriter.java | 12 +++++-------
.../store/file/operation/FileStoreWriteImpl.java | 8 +++-----
.../table/store/format/avro/AvroFileFormat.java | 8 ++++----
.../flink/table/store/format/orc/OrcFileFormat.java | 8 ++++----
9 files changed, 45 insertions(+), 52 deletions(-)
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormat.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormat.java
index 76c9cf9..6bf7d23 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormat.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormat.java
@@ -47,7 +47,11 @@ import java.util.ServiceLoader;
import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType;
-/** Factory class which creates reader and writer factories for specific file format. */
+/**
+ * Factory class which creates reader and writer factories for specific file format.
+ *
+ * <p>NOTE: This class must be thread safe.
+ */
public abstract class FileFormat {
protected abstract BulkDecodingFormat<RowData> getDecodingFormat();
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormatImpl.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormatImpl.java
index 0f8777c..35ba2fb 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormatImpl.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormatImpl.java
@@ -30,27 +30,27 @@ import org.apache.flink.table.factories.FactoryUtil;
/** A {@link FileFormat} which discovers reader and writer from format identifier. */
public class FileFormatImpl extends FileFormat {
- private final ClassLoader classLoader;
- private final String formatIdentifier;
+ private final BulkReaderFormatFactory readerFactory;
+ private final BulkWriterFormatFactory writerFactory;
private final ReadableConfig formatOptions;
public FileFormatImpl(
ClassLoader classLoader, String formatIdentifier, ReadableConfig formatOptions) {
- this.classLoader = classLoader;
- this.formatIdentifier = formatIdentifier;
+ this.readerFactory =
+ FactoryUtil.discoverFactory(
+ classLoader, BulkReaderFormatFactory.class, formatIdentifier);
+ this.writerFactory =
+ FactoryUtil.discoverFactory(
+ classLoader, BulkWriterFormatFactory.class, formatIdentifier);
this.formatOptions = formatOptions;
}
protected BulkDecodingFormat<RowData> getDecodingFormat() {
- return FactoryUtil.discoverFactory(
- classLoader, BulkReaderFormatFactory.class, formatIdentifier)
- .createDecodingFormat(null, formatOptions); // context is useless
+ return readerFactory.createDecodingFormat(null, formatOptions); // context is useless
}
@Override
protected EncodingFormat<BulkWriter.Factory<RowData>> getEncodingFormat() {
- return FactoryUtil.discoverFactory(
- classLoader, BulkWriterFormatFactory.class, formatIdentifier)
- .createEncodingFormat(null, formatOptions); // context is useless
+ return writerFactory.createEncodingFormat(null, formatOptions); // context is useless
}
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFile.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFile.java
index 49f69fe..0fa2132 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFile.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestFile.java
@@ -173,8 +173,7 @@ public class ManifestFile {
private final RowType partitionType;
private final RowType keyType;
private final RowType valueType;
- private final BulkFormat<RowData, FileSourceSplit> readerFactory;
- private final BulkWriter.Factory<RowData> writerFactory;
+ private final FileFormat fileFormat;
private final FileStorePathFactory pathFactory;
private final long suggestedFileSize;
@@ -188,19 +187,18 @@ public class ManifestFile {
this.partitionType = partitionType;
this.keyType = keyType;
this.valueType = valueType;
- RowType entryType = ManifestEntry.schema(partitionType, keyType, valueType);
- this.readerFactory = fileFormat.createReaderFactory(entryType);
- this.writerFactory = fileFormat.createWriterFactory(entryType);
+ this.fileFormat = fileFormat;
this.pathFactory = pathFactory;
this.suggestedFileSize = suggestedFileSize;
}
public ManifestFile create() {
+ RowType entryType = ManifestEntry.schema(partitionType, keyType, valueType);
return new ManifestFile(
partitionType,
new ManifestEntrySerializer(partitionType, keyType, valueType),
- readerFactory,
- writerFactory,
+ fileFormat.createReaderFactory(entryType),
+ fileFormat.createWriterFactory(entryType),
pathFactory,
suggestedFileSize);
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestList.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestList.java
index e15f546..82b64c0 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestList.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/manifest/ManifestList.java
@@ -103,24 +103,22 @@ public class ManifestList {
public static class Factory {
private final RowType partitionType;
- private final BulkFormat<RowData, FileSourceSplit> readerFactory;
- private final BulkWriter.Factory<RowData> writerFactory;
+ private final FileFormat fileFormat;
private final FileStorePathFactory pathFactory;
public Factory(
RowType partitionType, FileFormat fileFormat, FileStorePathFactory pathFactory) {
this.partitionType = partitionType;
- RowType metaType = ManifestFileMeta.schema(partitionType);
- this.readerFactory = fileFormat.createReaderFactory(metaType);
- this.writerFactory = fileFormat.createWriterFactory(metaType);
+ this.fileFormat = fileFormat;
this.pathFactory = pathFactory;
}
public ManifestList create() {
+ RowType metaType = ManifestFileMeta.schema(partitionType);
return new ManifestList(
new ManifestFileMetaSerializer(partitionType),
- readerFactory,
- writerFactory,
+ fileFormat.createReaderFactory(metaType),
+ fileFormat.createWriterFactory(metaType),
pathFactory);
}
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileReader.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileReader.java
index b3c0a59..4f56bf8 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileReader.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileReader.java
@@ -125,7 +125,6 @@ public class SstFileReader {
private int[][] valueProjection;
private RowType projectedKeyType;
private RowType projectedValueType;
- private BulkFormat<RowData, FileSourceSplit> readerFactory;
public Factory(
RowType keyType,
@@ -155,21 +154,19 @@ public class SstFileReader {
}
public SstFileReader create(BinaryRowData partition, int bucket) {
+ RowType recordType = KeyValue.schema(keyType, valueType);
+ int[][] projection =
+ KeyValue.project(keyProjection, valueProjection, keyType.getFieldCount());
return new SstFileReader(
projectedKeyType,
projectedValueType,
- readerFactory,
+ fileFormat.createReaderFactory(recordType, projection),
pathFactory.createSstPathFactory(partition, bucket));
}
private void applyProjection() {
projectedKeyType = (RowType) Projection.of(keyProjection).project(keyType);
projectedValueType = (RowType) Projection.of(valueProjection).project(valueType);
-
- RowType recordType = KeyValue.schema(keyType, valueType);
- int[][] projection =
- KeyValue.project(keyProjection, valueProjection, keyType.getFieldCount());
- readerFactory = fileFormat.createReaderFactory(recordType, projection);
}
}
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileWriter.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileWriter.java
index afc7e19..c72826a 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileWriter.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileWriter.java
@@ -265,8 +265,7 @@ public class SstFileWriter {
private final RowType keyType;
private final RowType valueType;
- private final BulkWriter.Factory<RowData> writerFactory;
- private final FileStatsExtractor fileStatsExtractor;
+ private final FileFormat fileFormat;
private final FileStorePathFactory pathFactory;
private final long suggestedFileSize;
@@ -278,19 +277,18 @@ public class SstFileWriter {
long suggestedFileSize) {
this.keyType = keyType;
this.valueType = valueType;
- RowType recordType = KeyValue.schema(keyType, valueType);
- this.writerFactory = fileFormat.createWriterFactory(recordType);
- this.fileStatsExtractor = fileFormat.createStatsExtractor(recordType).orElse(null);
+ this.fileFormat = fileFormat;
this.pathFactory = pathFactory;
this.suggestedFileSize = suggestedFileSize;
}
public SstFileWriter create(BinaryRowData partition, int bucket) {
+ RowType recordType = KeyValue.schema(keyType, valueType);
return new SstFileWriter(
keyType,
valueType,
- writerFactory,
- fileStatsExtractor,
+ fileFormat.createWriterFactory(recordType),
+ fileFormat.createStatsExtractor(recordType).orElse(null),
pathFactory.createSstPathFactory(partition, bucket),
suggestedFileSize);
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreWriteImpl.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreWriteImpl.java
index 2a47af1..a313c68 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreWriteImpl.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreWriteImpl.java
@@ -119,7 +119,7 @@ public class FileStoreWriteImpl implements FileStoreWrite {
sstFileWriter.valueType(),
options.writeBufferSize,
options.pageSize),
- createCompactManager(partition, bucket, sstFileWriter, compactExecutor),
+ createCompactManager(partition, bucket, compactExecutor),
new Levels(keyComparator, restoreFiles, options.numLevels),
maxSequenceNumber,
keyComparator,
@@ -129,15 +129,13 @@ public class FileStoreWriteImpl implements FileStoreWrite {
}
private CompactManager createCompactManager(
- BinaryRowData partition,
- int bucket,
- SstFileWriter sstFileWriter,
- ExecutorService compactExecutor) {
+ BinaryRowData partition, int bucket, ExecutorService compactExecutor) {
CompactStrategy compactStrategy =
new UniversalCompaction(
options.maxSizeAmplificationPercent,
options.sizeRatio,
options.numSortedRunMax);
+ SstFileWriter sstFileWriter = sstFileWriterFactory.create(partition, bucket);
CompactManager.Rewriter rewriter =
(outputLevel, dropDelete, sections) ->
sstFileWriter.write(
diff --git a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/avro/AvroFileFormat.java b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/avro/AvroFileFormat.java
index 10a6fa9..fcaed5c 100644
--- a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/avro/AvroFileFormat.java
+++ b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/avro/AvroFileFormat.java
@@ -28,21 +28,21 @@ import org.apache.flink.table.store.file.format.FileFormat;
/** Avro {@link FileFormat}. */
public class AvroFileFormat extends FileFormat {
+ private final org.apache.flink.formats.avro.AvroFileFormatFactory factory;
private final ReadableConfig formatOptions;
public AvroFileFormat(ReadableConfig formatOptions) {
+ this.factory = new org.apache.flink.formats.avro.AvroFileFormatFactory();
this.formatOptions = formatOptions;
}
@Override
protected BulkDecodingFormat<RowData> getDecodingFormat() {
- return new org.apache.flink.formats.avro.AvroFileFormatFactory()
- .createDecodingFormat(null, formatOptions); // context is useless
+ return factory.createDecodingFormat(null, formatOptions); // context is useless
}
@Override
protected EncodingFormat<BulkWriter.Factory<RowData>> getEncodingFormat() {
- return new org.apache.flink.formats.avro.AvroFileFormatFactory()
- .createEncodingFormat(null, formatOptions); // context is useless
+ return factory.createEncodingFormat(null, formatOptions); // context is useless
}
}
diff --git a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormat.java b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormat.java
index ddb6a97..e56eb05 100644
--- a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormat.java
+++ b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormat.java
@@ -32,22 +32,22 @@ import java.util.Optional;
/** Orc {@link FileFormat}. */
public class OrcFileFormat extends FileFormat {
+ private final org.apache.flink.orc.OrcFileFormatFactory factory;
private final ReadableConfig formatOptions;
public OrcFileFormat(ReadableConfig formatOptions) {
+ this.factory = new org.apache.flink.orc.OrcFileFormatFactory();
this.formatOptions = formatOptions;
}
@Override
protected BulkDecodingFormat<RowData> getDecodingFormat() {
- return new org.apache.flink.orc.OrcFileFormatFactory()
- .createDecodingFormat(null, formatOptions); // context is useless
+ return factory.createDecodingFormat(null, formatOptions); // context is useless
}
@Override
protected EncodingFormat<BulkWriter.Factory<RowData>> getEncodingFormat() {
- return new org.apache.flink.orc.OrcFileFormatFactory()
- .createEncodingFormat(null, formatOptions); // context is useless
+ return factory.createEncodingFormat(null, formatOptions); // context is useless
}
@Override
[flink-table-store] 12/15: [FLINK-27139] FileStoreCommitImpl#filterCommitted should exit when faced with expired files
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 2a527a6fe34cdea999aee15a546106a7b4c9c4f3
Author: tsreaper <ts...@gmail.com>
AuthorDate: Mon Apr 11 10:44:10 2022 +0800
[FLINK-27139] FileStoreCommitImpl#filterCommitted should exit when faced with expired files
This closes #84
---
.../table/store/file/operation/FileStoreCommitImpl.java | 14 ++++++++++++++
.../table/store/file/operation/FileStoreCommitTest.java | 17 +++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreCommitImpl.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreCommitImpl.java
index 75042a7..27ab159 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreCommitImpl.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreCommitImpl.java
@@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -117,6 +118,11 @@ public class FileStoreCommitImpl implements FileStoreCommit {
@Override
public List<ManifestCommittable> filterCommitted(List<ManifestCommittable> committableList) {
+ // nothing to filter, fast exit
+ if (committableList.isEmpty()) {
+ return committableList;
+ }
+
// if there is no previous snapshots then nothing should be filtered
Long latestSnapshotId = pathFactory.latestSnapshotId();
if (latestSnapshotId == null) {
@@ -131,6 +137,14 @@ public class FileStoreCommitImpl implements FileStoreCommit {
for (long id = latestSnapshotId; id >= Snapshot.FIRST_SNAPSHOT_ID; id--) {
Path snapshotPath = pathFactory.toSnapshotPath(id);
+ try {
+ if (!snapshotPath.getFileSystem().exists(snapshotPath)) {
+ // snapshots before this are expired
+ break;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot determine if snapshot #" + id + " exists.", e);
+ }
Snapshot snapshot = Snapshot.fromPath(snapshotPath);
if (commitUser.equals(snapshot.commitUser())) {
if (identifiers.containsKey(snapshot.commitIdentifier())) {
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/operation/FileStoreCommitTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/operation/FileStoreCommitTest.java
index ce1cb45..6f90e52 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/operation/FileStoreCommitTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/operation/FileStoreCommitTest.java
@@ -25,8 +25,11 @@ import org.apache.flink.table.store.file.Snapshot;
import org.apache.flink.table.store.file.TestFileStore;
import org.apache.flink.table.store.file.TestKeyValueGenerator;
import org.apache.flink.table.store.file.ValueKind;
+import org.apache.flink.table.store.file.manifest.ManifestCommittable;
import org.apache.flink.table.store.file.mergetree.compact.DeduplicateMergeFunction;
import org.apache.flink.table.store.file.utils.FailingAtomicRenameFileSystem;
+import org.apache.flink.table.store.file.utils.FileStorePathFactory;
+import org.apache.flink.table.store.file.utils.FileUtils;
import org.apache.flink.table.store.file.utils.SnapshotFinder;
import org.apache.flink.table.store.file.utils.TestAtomicRenameFileSystem;
@@ -41,6 +44,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -95,6 +99,19 @@ public class FileStoreCommitTest {
assertThat(SnapshotFinder.findLatest(snapshotDir)).isEqualTo(latestId);
}
+ @Test
+ public void testFilterCommittedAfterExpire() throws Exception {
+ testRandomConcurrentNoConflict(1, false);
+ // remove first snapshot to mimic expiration
+ TestFileStore store = createStore(false);
+ FileStorePathFactory pathFactory = store.pathFactory();
+ Path firstSnapshotPath = pathFactory.toSnapshotPath(Snapshot.FIRST_SNAPSHOT_ID);
+ FileUtils.deleteOrWarn(firstSnapshotPath);
+ // this test succeeds if this call does not fail
+ store.newCommit()
+ .filterCommitted(Collections.singletonList(new ManifestCommittable("dummy")));
+ }
+
protected void testRandomConcurrentNoConflict(int numThreads, boolean failing)
throws Exception {
// prepare test data
[flink-table-store] 11/15: [FLINK-27158] Adjust default value of compaction and compress
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 1c25a8d7bf1a798e87eea6d901c7f48eaa995b51
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Apr 11 10:42:46 2022 +0800
[FLINK-27158] Adjust default value of compaction and compress
This closes #86
---
.../flink/table/store/file/format/FileFormat.java | 5 +--
.../table/store/file/format/FileFormatFactory.java | 4 +-
.../store/file/mergetree/MergeTreeOptions.java | 8 ++--
.../FileStatsExtractingAvroFormatFactory.java | 4 +-
.../store/format/avro/AvroFileFormatFactory.java | 4 +-
.../table/store/format/orc/OrcFileFormat.java | 12 ++++--
.../store/format/orc/OrcFileFormatFactory.java | 21 ++++++++--
.../table/store/format/orc/OrcFileFormatTest.java | 48 ++++++++++++++++++++++
8 files changed, 87 insertions(+), 19 deletions(-)
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormat.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormat.java
index 6bf7d23..7e1ea5d 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormat.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormat.java
@@ -23,7 +23,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.DelegatingConfiguration;
-import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.connector.file.src.FileSourceSplit;
import org.apache.flink.connector.file.src.reader.BulkFormat;
import org.apache.flink.connector.file.table.format.BulkDecodingFormat;
@@ -108,14 +107,14 @@ public abstract class FileFormat {
Configuration tableOptions,
ConfigOption<String> formatOption) {
String formatIdentifier = tableOptions.get(formatOption);
- ReadableConfig formatOptions =
+ DelegatingConfiguration formatOptions =
new DelegatingConfiguration(tableOptions, formatIdentifier + ".");
return fromIdentifier(classLoader, formatIdentifier, formatOptions);
}
/** Create a {@link FileFormatImpl} from format identifier and format options. */
public static FileFormat fromIdentifier(
- ClassLoader classLoader, String formatIdentifier, ReadableConfig formatOptions) {
+ ClassLoader classLoader, String formatIdentifier, Configuration formatOptions) {
ServiceLoader<FileFormatFactory> serviceLoader =
ServiceLoader.load(FileFormatFactory.class);
for (FileFormatFactory factory : serviceLoader) {
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormatFactory.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormatFactory.java
index f103d2d..088c11c 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormatFactory.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/format/FileFormatFactory.java
@@ -18,12 +18,12 @@
package org.apache.flink.table.store.file.format;
-import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.configuration.Configuration;
/** Factory to create {@link FileFormat}. */
public interface FileFormatFactory {
String identifier();
- FileFormat create(ReadableConfig formatOptions);
+ FileFormat create(Configuration formatOptions);
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/MergeTreeOptions.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/MergeTreeOptions.java
index 4cd6170..da2ff87 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/MergeTreeOptions.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/MergeTreeOptions.java
@@ -59,7 +59,7 @@ public class MergeTreeOptions {
public static final ConfigOption<Integer> NUM_LEVELS =
ConfigOptions.key("num-levels")
.intType()
- .defaultValue(4)
+ .noDefaultValue()
.withDescription(
"Total level number, for example, there are 3 levels, including 0,1,2 levels.");
@@ -107,7 +107,7 @@ public class MergeTreeOptions {
long pageSize,
long targetFileSize,
int numSortedRunMax,
- int numLevels,
+ Integer numLevels,
boolean commitForceCompact,
int maxSizeAmplificationPercent,
int sizeRatio) {
@@ -115,7 +115,9 @@ public class MergeTreeOptions {
this.pageSize = pageSize;
this.targetFileSize = targetFileSize;
this.numSortedRunMax = numSortedRunMax;
- this.numLevels = numLevels;
+ // By default, this ensures that the compaction does not fall to level 0, but at least to
+ // level 1
+ this.numLevels = numLevels == null ? numSortedRunMax + 1 : numLevels;
this.commitForceCompact = commitForceCompact;
this.maxSizeAmplificationPercent = maxSizeAmplificationPercent;
this.sizeRatio = sizeRatio;
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/format/FileStatsExtractingAvroFormatFactory.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/format/FileStatsExtractingAvroFormatFactory.java
index 656fb0b..9c84176 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/format/FileStatsExtractingAvroFormatFactory.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/format/FileStatsExtractingAvroFormatFactory.java
@@ -18,7 +18,7 @@
package org.apache.flink.table.store.file.format;
-import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.configuration.Configuration;
/** Factory to create {@link FileStatsExtractingAvroFormat}. */
public class FileStatsExtractingAvroFormatFactory implements FileFormatFactory {
@@ -29,7 +29,7 @@ public class FileStatsExtractingAvroFormatFactory implements FileFormatFactory {
}
@Override
- public FileFormat create(ReadableConfig formatOptions) {
+ public FileFormat create(Configuration formatOptions) {
return new FileStatsExtractingAvroFormat();
}
}
diff --git a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/avro/AvroFileFormatFactory.java b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/avro/AvroFileFormatFactory.java
index 63a0e45..26c5ad5 100644
--- a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/avro/AvroFileFormatFactory.java
+++ b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/avro/AvroFileFormatFactory.java
@@ -18,7 +18,7 @@
package org.apache.flink.table.store.format.avro;
-import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.table.store.file.format.FileFormat;
import org.apache.flink.table.store.file.format.FileFormatFactory;
@@ -31,7 +31,7 @@ public class AvroFileFormatFactory implements FileFormatFactory {
}
@Override
- public FileFormat create(ReadableConfig formatOptions) {
+ public FileFormat create(Configuration formatOptions) {
return new AvroFileFormat(formatOptions);
}
}
diff --git a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormat.java b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormat.java
index e56eb05..477ab41 100644
--- a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormat.java
+++ b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormat.java
@@ -18,8 +18,9 @@
package org.apache.flink.table.store.format.orc;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.serialization.BulkWriter;
-import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.file.table.format.BulkDecodingFormat;
import org.apache.flink.table.connector.format.EncodingFormat;
import org.apache.flink.table.data.RowData;
@@ -33,13 +34,18 @@ import java.util.Optional;
public class OrcFileFormat extends FileFormat {
private final org.apache.flink.orc.OrcFileFormatFactory factory;
- private final ReadableConfig formatOptions;
+ private final Configuration formatOptions;
- public OrcFileFormat(ReadableConfig formatOptions) {
+ public OrcFileFormat(Configuration formatOptions) {
this.factory = new org.apache.flink.orc.OrcFileFormatFactory();
this.formatOptions = formatOptions;
}
+ @VisibleForTesting
+ Configuration formatOptions() {
+ return formatOptions;
+ }
+
@Override
protected BulkDecodingFormat<RowData> getDecodingFormat() {
return factory.createDecodingFormat(null, formatOptions); // context is useless
diff --git a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormatFactory.java b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormatFactory.java
index b0902be..9e544e4 100644
--- a/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormatFactory.java
+++ b/flink-table-store-format/src/main/java/org/apache/flink/table/store/format/orc/OrcFileFormatFactory.java
@@ -18,10 +18,11 @@
package org.apache.flink.table.store.format.orc;
-import org.apache.flink.configuration.ReadableConfig;
-import org.apache.flink.table.store.file.format.FileFormat;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.table.store.file.format.FileFormatFactory;
+import java.util.Properties;
+
/** Factory to create {@link OrcFileFormat}. */
public class OrcFileFormatFactory implements FileFormatFactory {
@@ -31,7 +32,19 @@ public class OrcFileFormatFactory implements FileFormatFactory {
}
@Override
- public FileFormat create(ReadableConfig formatOptions) {
- return new OrcFileFormat(formatOptions);
+ public OrcFileFormat create(Configuration formatOptions) {
+ return new OrcFileFormat(supplyDefaultOptions(formatOptions));
+ }
+
+ private Configuration supplyDefaultOptions(Configuration options) {
+ if (!options.containsKey("compress")) {
+ Properties properties = new Properties();
+ options.addAllToProperties(properties);
+ properties.setProperty("compress", "lz4");
+ Configuration newOptions = new Configuration();
+ properties.forEach((k, v) -> newOptions.setString(k.toString(), v.toString()));
+ return newOptions;
+ }
+ return options;
}
}
diff --git a/flink-table-store-format/src/test/java/org/apache/flink/table/store/format/orc/OrcFileFormatTest.java b/flink-table-store-format/src/test/java/org/apache/flink/table/store/format/orc/OrcFileFormatTest.java
new file mode 100644
index 0000000..b99d337
--- /dev/null
+++ b/flink-table-store-format/src/test/java/org/apache/flink/table/store/format/orc/OrcFileFormatTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.format.orc;
+
+import org.apache.flink.configuration.Configuration;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for {@link OrcFileFormatFactory}. */
+public class OrcFileFormatTest {
+
+ @Test
+ public void testAbsent() {
+ Configuration options = new Configuration();
+ options.setString("haha", "1");
+ OrcFileFormat orc = new OrcFileFormatFactory().create(options);
+ assertThat(orc.formatOptions().getString("haha", "")).isEqualTo("1");
+ assertThat(orc.formatOptions().getString("compress", "")).isEqualTo("lz4");
+ }
+
+ @Test
+ public void testPresent() {
+ Configuration options = new Configuration();
+ options.setString("haha", "1");
+ options.setString("compress", "zlib");
+ OrcFileFormat orc = new OrcFileFormatFactory().create(options);
+ assertThat(orc.formatOptions().getString("haha", "")).isEqualTo("1");
+ assertThat(orc.formatOptions().getString("compress", "")).isEqualTo("zlib");
+ }
+}
[flink-table-store] 02/15: [FLINK-27085] Introduce snapshot.num-retained.min option
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit d466ec7e0a29fbdabcf4c2ff87478def46524f5a
Author: tsreaper <ts...@gmail.com>
AuthorDate: Wed Apr 6 18:05:53 2022 +0800
[FLINK-27085] Introduce snapshot.num-retained.min option
This closes #75
---
docs/content/docs/development/write-table.md | 9 ++++++-
.../flink/table/store/file/FileStoreImpl.java | 3 ++-
.../flink/table/store/file/FileStoreOptions.java | 29 ++++++++++++++++++----
.../store/file/operation/FileStoreExpireImpl.java | 17 +++++++------
.../flink/table/store/file/TestFileStore.java | 6 +++--
.../store/file/operation/FileStoreExpireTest.java | 27 ++++++++++++--------
6 files changed, 64 insertions(+), 27 deletions(-)
diff --git a/docs/content/docs/development/write-table.md b/docs/content/docs/development/write-table.md
index 7b7c7ad..b866737 100644
--- a/docs/content/docs/development/write-table.md
+++ b/docs/content/docs/development/write-table.md
@@ -142,7 +142,14 @@ to eliminate expired snapshots:
<td>The maximum time of completed snapshots to retain.</td>
</tr>
<tr>
- <td><h5>snapshot.num-retained</h5></td>
+ <td><h5>snapshot.num-retained.min</h5></td>
+ <td>No</td>
+ <td style="word-wrap: break-word;">10</td>
+ <td>Integer</td>
+ <td>The minimum number of completed snapshots to retain.</td>
+ </tr>
+ <tr>
+ <td><h5>snapshot.num-retained.max</h5></td>
<td>No</td>
<td style="word-wrap: break-word;">Integer.MAX_VALUE</td>
<td>Integer</td>
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreImpl.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreImpl.java
index 72e7c50..71397b2 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreImpl.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreImpl.java
@@ -135,7 +135,8 @@ public class FileStoreImpl implements FileStore {
@Override
public FileStoreExpireImpl newExpire() {
return new FileStoreExpireImpl(
- options.snapshotNumRetain(),
+ options.snapshotNumRetainMin(),
+ options.snapshotNumRetainMax(),
options.snapshotTimeRetain().toMillis(),
pathFactory(),
manifestFileFactory(),
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreOptions.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreOptions.java
index 7f7bf8f..f9fbd39 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreOptions.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreOptions.java
@@ -87,8 +87,14 @@ public class FileStoreOptions implements Serializable {
"The default partition name in case the dynamic partition"
+ " column value is null/empty string.");
- public static final ConfigOption<Integer> SNAPSHOT_NUM_RETAINED =
- ConfigOptions.key("snapshot.num-retained")
+ public static final ConfigOption<Integer> SNAPSHOT_NUM_RETAINED_MIN =
+ ConfigOptions.key("snapshot.num-retained.min")
+ .intType()
+ .defaultValue(10)
+ .withDescription("The minimum number of completed snapshots to retain.");
+
+ public static final ConfigOption<Integer> SNAPSHOT_NUM_RETAINED_MAX =
+ ConfigOptions.key("snapshot.num-retained.max")
.intType()
.defaultValue(Integer.MAX_VALUE)
.withDescription("The maximum number of completed snapshots to retain.");
@@ -116,7 +122,8 @@ public class FileStoreOptions implements Serializable {
allOptions.add(MANIFEST_TARGET_FILE_SIZE);
allOptions.add(MANIFEST_MERGE_MIN_COUNT);
allOptions.add(PARTITION_DEFAULT_NAME);
- allOptions.add(SNAPSHOT_NUM_RETAINED);
+ allOptions.add(SNAPSHOT_NUM_RETAINED_MIN);
+ allOptions.add(SNAPSHOT_NUM_RETAINED_MAX);
allOptions.add(SNAPSHOT_TIME_RETAINED);
return allOptions;
}
@@ -124,6 +131,14 @@ public class FileStoreOptions implements Serializable {
public FileStoreOptions(Configuration options) {
this.options = options;
// TODO validate all keys
+ Preconditions.checkArgument(
+ snapshotNumRetainMin() > 0,
+ SNAPSHOT_NUM_RETAINED_MIN.key() + " should be at least 1");
+ Preconditions.checkArgument(
+ snapshotNumRetainMin() <= snapshotNumRetainMax(),
+ SNAPSHOT_NUM_RETAINED_MIN.key()
+ + " should not be larger than "
+ + SNAPSHOT_NUM_RETAINED_MAX.key());
}
public int bucket() {
@@ -177,8 +192,12 @@ public class FileStoreOptions implements Serializable {
return new MergeTreeOptions(options);
}
- public int snapshotNumRetain() {
- return options.get(SNAPSHOT_NUM_RETAINED);
+ public int snapshotNumRetainMin() {
+ return options.get(SNAPSHOT_NUM_RETAINED_MIN);
+ }
+
+ public int snapshotNumRetainMax() {
+ return options.get(SNAPSHOT_NUM_RETAINED_MAX);
}
public Duration snapshotTimeRetain() {
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreExpireImpl.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreExpireImpl.java
index 7d7d94c..feda420 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreExpireImpl.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreExpireImpl.java
@@ -52,8 +52,9 @@ public class FileStoreExpireImpl implements FileStoreExpire {
private static final Logger LOG = LoggerFactory.getLogger(FileStoreExpireImpl.class);
+ private final int numRetainedMin;
// snapshots exceeding any constraint will be expired
- private final int numRetained;
+ private final int numRetainedMax;
private final long millisRetained;
private final FileStorePathFactory pathFactory;
@@ -63,12 +64,14 @@ public class FileStoreExpireImpl implements FileStoreExpire {
private Lock lock;
public FileStoreExpireImpl(
- int numRetained,
+ int numRetainedMin,
+ int numRetainedMax,
long millisRetained,
FileStorePathFactory pathFactory,
ManifestFile.Factory manifestFileFactory,
ManifestList.Factory manifestListFactory) {
- this.numRetained = numRetained;
+ this.numRetainedMin = numRetainedMin;
+ this.numRetainedMax = numRetainedMax;
this.millisRetained = millisRetained;
this.pathFactory = pathFactory;
this.manifestFile = manifestFileFactory.create();
@@ -102,8 +105,8 @@ public class FileStoreExpireImpl implements FileStoreExpire {
}
// find the earliest snapshot to retain
- for (long id = Math.max(latestSnapshotId - numRetained + 1, earliest);
- id <= latestSnapshotId;
+ for (long id = Math.max(latestSnapshotId - numRetainedMax + 1, earliest);
+ id <= latestSnapshotId - numRetainedMin;
id++) {
Path snapshotPath = pathFactory.toSnapshotPath(id);
try {
@@ -121,8 +124,8 @@ public class FileStoreExpireImpl implements FileStoreExpire {
}
}
- // no snapshot can be retained, expire all but last one
- expireUntil(earliest, latestSnapshotId);
+ // no snapshot can be retained, expire until there are only numRetainedMin snapshots left
+ expireUntil(earliest, latestSnapshotId - numRetainedMin + 1);
}
private void expireUntil(long earliestId, long endExclusiveId) {
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/TestFileStore.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/TestFileStore.java
index 15a85d0..1c5bbbf 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/TestFileStore.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/TestFileStore.java
@@ -122,9 +122,11 @@ public class TestFileStore extends FileStoreImpl {
this.valueSerializer = new RowDataSerializer(valueType);
}
- public FileStoreExpireImpl newExpire(int numRetained, long millisRetained) {
+ public FileStoreExpireImpl newExpire(
+ int numRetainedMin, int numRetainedMax, long millisRetained) {
return new FileStoreExpireImpl(
- numRetained,
+ numRetainedMin,
+ numRetainedMax,
millisRetained,
pathFactory(),
manifestFileFactory(),
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/operation/FileStoreExpireTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/operation/FileStoreExpireTest.java
index 2884a85..84da568 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/operation/FileStoreExpireTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/operation/FileStoreExpireTest.java
@@ -74,7 +74,7 @@ public class FileStoreExpireTest {
@Test
public void testNoSnapshot() {
- FileStoreExpire expire = store.newExpire(3, Long.MAX_VALUE);
+ FileStoreExpire expire = store.newExpire(1, 3, Long.MAX_VALUE);
expire.expire();
assertThat(pathFactory.latestSnapshotId()).isNull();
@@ -86,7 +86,7 @@ public class FileStoreExpireTest {
List<Integer> snapshotPositions = new ArrayList<>();
commit(2, allData, snapshotPositions);
int latestSnapshotId = pathFactory.latestSnapshotId().intValue();
- FileStoreExpire expire = store.newExpire(latestSnapshotId + 1, Long.MAX_VALUE);
+ FileStoreExpire expire = store.newExpire(1, latestSnapshotId + 1, Long.MAX_VALUE);
expire.expire();
FileSystem fs = pathFactory.toSnapshotPath(latestSnapshotId).getFileSystem();
@@ -102,7 +102,7 @@ public class FileStoreExpireTest {
List<Integer> snapshotPositions = new ArrayList<>();
commit(5, allData, snapshotPositions);
int latestSnapshotId = pathFactory.latestSnapshotId().intValue();
- FileStoreExpire expire = store.newExpire(Integer.MAX_VALUE, Long.MAX_VALUE);
+ FileStoreExpire expire = store.newExpire(1, Integer.MAX_VALUE, Long.MAX_VALUE);
expire.expire();
FileSystem fs = pathFactory.toSnapshotPath(latestSnapshotId).getFileSystem();
@@ -113,26 +113,31 @@ public class FileStoreExpireTest {
}
@Test
- public void testKeepAtLeastOneSnapshot() throws Exception {
+ public void testNumRetainedMin() throws Exception {
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+
+ int numRetainedMin = random.nextInt(5) + 1;
List<KeyValue> allData = new ArrayList<>();
List<Integer> snapshotPositions = new ArrayList<>();
- commit(3, allData, snapshotPositions);
+ commit(numRetainedMin + random.nextInt(5), allData, snapshotPositions);
int latestSnapshotId = pathFactory.latestSnapshotId().intValue();
Thread.sleep(100);
- FileStoreExpire expire = store.newExpire(Integer.MAX_VALUE, 1);
+ FileStoreExpire expire = store.newExpire(numRetainedMin, Integer.MAX_VALUE, 1);
expire.expire();
FileSystem fs = pathFactory.toSnapshotPath(latestSnapshotId).getFileSystem();
- for (int i = 1; i < latestSnapshotId; i++) {
+ for (int i = 1; i <= latestSnapshotId - numRetainedMin; i++) {
assertThat(fs.exists(pathFactory.toSnapshotPath(i))).isFalse();
}
- assertThat(fs.exists(pathFactory.toSnapshotPath(latestSnapshotId))).isTrue();
- assertSnapshot(latestSnapshotId, allData, snapshotPositions);
+ for (int i = latestSnapshotId - numRetainedMin + 1; i <= latestSnapshotId; i++) {
+ assertThat(fs.exists(pathFactory.toSnapshotPath(i))).isTrue();
+ assertSnapshot(i, allData, snapshotPositions);
+ }
}
@Test
public void testExpireWithNumber() throws Exception {
- FileStoreExpire expire = store.newExpire(3, Long.MAX_VALUE);
+ FileStoreExpire expire = store.newExpire(1, 3, Long.MAX_VALUE);
List<KeyValue> allData = new ArrayList<>();
List<Integer> snapshotPositions = new ArrayList<>();
@@ -169,7 +174,7 @@ public class FileStoreExpireTest {
@Test
public void testExpireWithTime() throws Exception {
- FileStoreExpire expire = store.newExpire(Integer.MAX_VALUE, 1000);
+ FileStoreExpire expire = store.newExpire(1, Integer.MAX_VALUE, 1000);
List<KeyValue> allData = new ArrayList<>();
List<Integer> snapshotPositions = new ArrayList<>();
[flink-table-store] 14/15: [hotfix] Add logs to file store compact for better tunning
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit d27c9345f9661099bf21221c00c4f4a0c08d97db
Author: tsreaper <ts...@gmail.com>
AuthorDate: Mon Apr 11 15:35:18 2022 +0800
[hotfix] Add logs to file store compact for better tunning
This closes #88
---
.../file/mergetree/compact/CompactManager.java | 75 +++++++++++++++++++++-
.../mergetree/compact/UniversalCompaction.java | 14 ++++
2 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/compact/CompactManager.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/compact/CompactManager.java
index 4487d3f..0d1c1d3 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/compact/CompactManager.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/compact/CompactManager.java
@@ -23,6 +23,9 @@ import org.apache.flink.table.store.file.mergetree.Levels;
import org.apache.flink.table.store.file.mergetree.SortedRun;
import org.apache.flink.table.store.file.mergetree.sst.SstFileMeta;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@@ -31,12 +34,15 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.stream.Collectors;
import static java.util.Collections.singletonList;
/** Manager to submit compaction task. */
public class CompactManager {
+ private static final Logger LOG = LoggerFactory.getLogger(CompactManager.class);
+
private final ExecutorService executor;
private final CompactStrategy strategy;
@@ -84,6 +90,31 @@ public class CompactManager {
boolean dropDelete =
unit.outputLevel() != 0
&& unit.outputLevel() >= levels.nonEmptyHighestLevel();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Submit compaction with files (level, size): "
+ + levels.levelSortedRuns().stream()
+ .flatMap(lsr -> lsr.run().files().stream())
+ .map(
+ file ->
+ String.format(
+ "(%d, %d)",
+ file.level(),
+ file.fileSize()))
+ .collect(Collectors.joining(", ")));
+ LOG.debug(
+ "Pick these files (level, size) for compaction: "
+ + unit.files().stream()
+ .map(
+ file ->
+ String.format(
+ "(%d, %d)",
+ file.level(),
+ file.fileSize()))
+ .collect(Collectors.joining(", ")));
+ }
+
CompactTask task = new CompactTask(unit, dropDelete);
taskFuture = executor.submit(task);
});
@@ -131,10 +162,21 @@ public class CompactManager {
private final boolean dropDelete;
+ // metrics
+ private long rewriteInputSize;
+ private long rewriteOutputSize;
+ private int rewriteFilesNum;
+ private int upgradeFilesNum;
+
private CompactTask(CompactUnit unit, boolean dropDelete) {
this.outputLevel = unit.outputLevel();
this.partitioned = new IntervalPartition(unit.files(), keyComparator).partition();
this.dropDelete = dropDelete;
+
+ this.rewriteInputSize = 0;
+ this.rewriteOutputSize = 0;
+ this.rewriteFilesNum = 0;
+ this.upgradeFilesNum = 0;
}
@Override
@@ -143,6 +185,8 @@ public class CompactManager {
}
private CompactResult compact() throws Exception {
+ long startMillis = System.currentTimeMillis();
+
List<List<SortedRun>> candidate = new ArrayList<>();
List<SstFileMeta> before = new ArrayList<>();
List<SstFileMeta> after = new ArrayList<>();
@@ -172,6 +216,20 @@ public class CompactManager {
}
}
rewrite(candidate, before, after);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Done compacting {} files to {} files in {}ms. "
+ + "Rewrite input size = {}, output size = {}, rewrite file num = {}, upgrade file num = {}",
+ before.size(),
+ after.size(),
+ System.currentTimeMillis() - startMillis,
+ rewriteInputSize,
+ rewriteOutputSize,
+ rewriteFilesNum,
+ upgradeFilesNum);
+ }
+
return result(before, after);
}
@@ -179,6 +237,7 @@ public class CompactManager {
if (file.level() != outputLevel) {
before.add(file);
after.add(file.upgrade(outputLevel));
+ upgradeFilesNum++;
}
}
@@ -200,8 +259,20 @@ public class CompactManager {
return;
}
}
- candidate.forEach(runs -> runs.forEach(run -> before.addAll(run.files())));
- after.addAll(rewriter.rewrite(outputLevel, dropDelete, candidate));
+ candidate.forEach(
+ runs ->
+ runs.forEach(
+ run -> {
+ before.addAll(run.files());
+ rewriteInputSize +=
+ run.files().stream()
+ .mapToLong(SstFileMeta::fileSize)
+ .sum();
+ rewriteFilesNum += run.files().size();
+ }));
+ List<SstFileMeta> result = rewriter.rewrite(outputLevel, dropDelete, candidate);
+ after.addAll(result);
+ rewriteOutputSize += result.stream().mapToLong(SstFileMeta::fileSize).sum();
candidate.clear();
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/compact/UniversalCompaction.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/compact/UniversalCompaction.java
index d37c899..d9e2a4e 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/compact/UniversalCompaction.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/compact/UniversalCompaction.java
@@ -22,6 +22,9 @@ import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.table.store.file.mergetree.LevelSortedRun;
import org.apache.flink.table.store.file.mergetree.SortedRun;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.List;
import java.util.Optional;
@@ -34,6 +37,8 @@ import java.util.Optional;
*/
public class UniversalCompaction implements CompactStrategy {
+ private static final Logger LOG = LoggerFactory.getLogger(UniversalCompaction.class);
+
private final int maxSizeAmp;
private final int sizeRatio;
private final int maxRunNum;
@@ -51,12 +56,18 @@ public class UniversalCompaction implements CompactStrategy {
// 1 checking for reducing size amplification
CompactUnit unit = pickForSizeAmp(maxLevel, runs);
if (unit != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Universal compaction due to size amplification");
+ }
return Optional.of(unit);
}
// 2 checking for size ratio
unit = pickForSizeRatio(maxLevel, runs);
if (unit != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Universal compaction due to size ratio");
+ }
return Optional.of(unit);
}
@@ -64,6 +75,9 @@ public class UniversalCompaction implements CompactStrategy {
if (runs.size() > maxRunNum) {
// compacting for file num
int candidateCount = runs.size() - maxRunNum + 1;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Universal compaction due to file num");
+ }
return Optional.ofNullable(pickForSizeRatio(maxLevel, runs, candidateCount));
}
[flink-table-store] 09/15: [FLINK-27135] Rename 'file.path' table option in table store to 'path'
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 2385f68c2619bdac53a7956f4512d086c36ff45e
Author: tsreaper <ts...@gmail.com>
AuthorDate: Fri Apr 8 17:44:07 2022 +0800
[FLINK-27135] Rename 'file.path' table option in table store to 'path'
This closes #82
---
docs/content/docs/development/create-table.md | 4 ++--
docs/content/docs/try-table-store/quick-start.md | 2 +-
.../table/store/connector/ContinuousFileStoreITCase.java | 4 ++--
.../apache/flink/table/store/connector/FileStoreITCase.java | 6 +++---
.../flink/table/store/connector/ReadWriteTableITCase.java | 2 +-
.../flink/table/store/connector/ReadWriteTableTestBase.java | 2 +-
.../flink/table/store/connector/TableStoreFactoryTest.java | 10 +++++-----
.../flink/table/store/connector/TableStoreTestBase.java | 4 ++--
.../org/apache/flink/table/store/file/FileStoreOptions.java | 12 ++++++------
.../org/apache/flink/table/store/file/TestFileStore.java | 4 ++--
10 files changed, 25 insertions(+), 25 deletions(-)
diff --git a/docs/content/docs/development/create-table.md b/docs/content/docs/development/create-table.md
index 106afe3..8b94f4f 100644
--- a/docs/content/docs/development/create-table.md
+++ b/docs/content/docs/development/create-table.md
@@ -106,7 +106,7 @@ Important options include the following:
</thead>
<tbody>
<tr>
- <td><h5>file.path</h5></td>
+ <td><h5>path</h5></td>
<td>Yes</td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
@@ -147,7 +147,7 @@ Important options include the following:
Creating a table will create the corresponding physical storage:
- The table's FileStore directory will be created under:
- `${file.path}/${catalog_name}.catalog/${database_name}.db/${table_name}`
+ `${path}/${catalog_name}.catalog/${database_name}.db/${table_name}`
- If `log.system` is configured as Kafka, a Topic named
"${catalog_name}.${database_name}.${table_name}" will be created
automatically when the table is created.
diff --git a/docs/content/docs/try-table-store/quick-start.md b/docs/content/docs/try-table-store/quick-start.md
index db7ca2a..3c00f5b 100644
--- a/docs/content/docs/try-table-store/quick-start.md
+++ b/docs/content/docs/try-table-store/quick-start.md
@@ -112,7 +112,7 @@ Start the SQL Client CLI:
```sql
-- set root path to session config
-SET 'table-store.file.path' = '/tmp/table_store';
+SET 'table-store.path' = '/tmp/table_store';
-- create a word count dynamic table without 'connector' option
CREATE TABLE word_count (
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ContinuousFileStoreITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ContinuousFileStoreITCase.java
index 8c10ab9..f873e8e 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ContinuousFileStoreITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ContinuousFileStoreITCase.java
@@ -35,7 +35,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import static org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL;
-import static org.apache.flink.table.store.file.FileStoreOptions.FILE_PATH;
+import static org.apache.flink.table.store.file.FileStoreOptions.PATH;
import static org.apache.flink.table.store.file.FileStoreOptions.TABLE_STORE_PREFIX;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -59,7 +59,7 @@ public class ContinuousFileStoreITCase extends AbstractTestBase {
private void prepareEnv(TableEnvironment env, String path) {
Configuration config = env.getConfig().getConfiguration();
config.set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 2);
- config.setString(TABLE_STORE_PREFIX + FILE_PATH.key(), path);
+ config.setString(TABLE_STORE_PREFIX + PATH.key(), path);
env.executeSql("CREATE TABLE IF NOT EXISTS T1 (a STRING, b STRING, c STRING)");
env.executeSql(
"CREATE TABLE IF NOT EXISTS T2 (a STRING, b STRING, c STRING, PRIMARY KEY (a) NOT ENFORCED)");
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
index 645e208..c6b7d96 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
@@ -63,7 +63,7 @@ import java.util.stream.Stream;
import static org.apache.flink.table.store.file.FileStoreOptions.BUCKET;
import static org.apache.flink.table.store.file.FileStoreOptions.FILE_FORMAT;
-import static org.apache.flink.table.store.file.FileStoreOptions.FILE_PATH;
+import static org.apache.flink.table.store.file.FileStoreOptions.PATH;
import static org.assertj.core.api.Assertions.assertThat;
/** ITCase for {@link FileStoreSource} and {@link StoreSink}. */
@@ -347,10 +347,10 @@ public class FileStoreITCase extends AbstractTestBase {
Configuration options = new Configuration();
options.set(BUCKET, NUM_BUCKET);
if (noFail) {
- options.set(FILE_PATH, folder.toURI().toString());
+ options.set(PATH, folder.toURI().toString());
} else {
FailingAtomicRenameFileSystem.get().reset(3, 100);
- options.set(FILE_PATH, FailingAtomicRenameFileSystem.getFailingPath(folder.getPath()));
+ options.set(PATH, FailingAtomicRenameFileSystem.getFailingPath(folder.getPath()));
}
options.set(FILE_FORMAT, "avro");
return options;
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
index bff091c..f04fb5e 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
@@ -1214,7 +1214,7 @@ public class ReadWriteTableITCase extends ReadWriteTableTestBase {
id));
tEnv.executeSql(
String.format(
- "create table managed_table with ('file.path' = '%s') "
+ "create table managed_table with ('path' = '%s') "
+ "like dummy_source (excluding options)",
rootPath));
tEnv.executeSql("insert into managed_table select * from dummy_source").await();
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
index ad54b50..5c47b7a 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
@@ -288,7 +288,7 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
throws Exception {
Map<String, String> tableOptions = new HashMap<>();
rootPath = TEMPORARY_FOLDER.newFolder().getPath();
- tableOptions.put(FileStoreOptions.FILE_PATH.key(), rootPath);
+ tableOptions.put(FileStoreOptions.PATH.key(), rootPath);
if (enableLogStore) {
tableOptions.put(LOG_SYSTEM.key(), "kafka");
tableOptions.put(LOG_PREFIX + BOOTSTRAP_SERVERS.key(), getBootstrapServers());
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreFactoryTest.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreFactoryTest.java
index 4385050..b32fb24 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreFactoryTest.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreFactoryTest.java
@@ -50,7 +50,7 @@ import java.util.stream.Stream;
import static org.apache.flink.table.store.connector.TableStoreTestBase.createResolvedTable;
import static org.apache.flink.table.store.file.FileStoreOptions.BUCKET;
-import static org.apache.flink.table.store.file.FileStoreOptions.FILE_PATH;
+import static org.apache.flink.table.store.file.FileStoreOptions.PATH;
import static org.apache.flink.table.store.file.FileStoreOptions.TABLE_STORE_PREFIX;
import static org.apache.flink.table.store.file.FileStoreOptions.relativeTablePath;
import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
@@ -202,7 +202,7 @@ public class TableStoreFactoryTest {
of(
BUCKET.key(),
BUCKET.defaultValue().toString(),
- FILE_PATH.key(),
+ PATH.key(),
sharedTempDir.toString(),
LOG_PREFIX + BOOTSTRAP_SERVERS.key(),
"localhost:9092",
@@ -226,7 +226,7 @@ public class TableStoreFactoryTest {
// set both session and table level configuration to test options combination
Map<String, String> tableOptions = new HashMap<>(enrichedOptions);
- tableOptions.remove(FILE_PATH.key());
+ tableOptions.remove(PATH.key());
tableOptions.remove(CONSISTENCY.key());
Arguments arg3 =
Arguments.of(
@@ -247,7 +247,7 @@ public class TableStoreFactoryTest {
private static Stream<Arguments> providingEnrichedOptionsForCreation() {
Map<String, String> enrichedOptions = new HashMap<>();
- enrichedOptions.put(FILE_PATH.key(), sharedTempDir.toAbsolutePath().toString());
+ enrichedOptions.put(PATH.key(), sharedTempDir.toAbsolutePath().toString());
return Stream.of(
Arguments.of(enrichedOptions, false),
Arguments.of(enrichedOptions, true),
@@ -264,7 +264,7 @@ public class TableStoreFactoryTest {
tablePath.mkdirs();
}
Map<String, String> enrichedOptions = new HashMap<>();
- enrichedOptions.put(FILE_PATH.key(), sharedTempDir.toAbsolutePath().toString());
+ enrichedOptions.put(PATH.key(), sharedTempDir.toAbsolutePath().toString());
return Stream.of(
Arguments.of(enrichedOptions, false),
Arguments.of(enrichedOptions, true),
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreTestBase.java
index 1bba583..c6762da 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreTestBase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/TableStoreTestBase.java
@@ -51,7 +51,7 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.apache.flink.table.store.connector.TableStoreFactoryOptions.LOG_SYSTEM;
-import static org.apache.flink.table.store.file.FileStoreOptions.FILE_PATH;
+import static org.apache.flink.table.store.file.FileStoreOptions.PATH;
import static org.apache.flink.table.store.file.FileStoreOptions.TABLE_STORE_PREFIX;
import static org.apache.flink.table.store.kafka.KafkaLogOptions.BOOTSTRAP_SERVERS;
import static org.apache.flink.table.store.log.LogOptions.LOG_PREFIX;
@@ -109,7 +109,7 @@ public abstract class TableStoreTestBase extends KafkaTableTestBase {
protected void prepareSessionContext() {
Configuration configuration = tEnv.getConfig().getConfiguration();
- configuration.setString(TABLE_STORE_PREFIX + FILE_PATH.key(), rootPath);
+ configuration.setString(TABLE_STORE_PREFIX + PATH.key(), rootPath);
configuration.setString(
TABLE_STORE_PREFIX + LOG_PREFIX + BOOTSTRAP_SERVERS.key(), getBootstrapServers());
if (enableLogStore) {
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreOptions.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreOptions.java
index f9fbd39..0b91c9b 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreOptions.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/FileStoreOptions.java
@@ -47,8 +47,8 @@ public class FileStoreOptions implements Serializable {
.defaultValue(1)
.withDescription("Bucket number for file store.");
- public static final ConfigOption<String> FILE_PATH =
- ConfigOptions.key("file.path")
+ public static final ConfigOption<String> PATH =
+ ConfigOptions.key("path")
.stringType()
.noDefaultValue()
.withDescription("The root file path of the table store in the filesystem.");
@@ -116,7 +116,7 @@ public class FileStoreOptions implements Serializable {
public static Set<ConfigOption<?>> allOptions() {
Set<ConfigOption<?>> allOptions = new HashSet<>();
allOptions.add(BUCKET);
- allOptions.add(FILE_PATH);
+ allOptions.add(PATH);
allOptions.add(FILE_FORMAT);
allOptions.add(MANIFEST_FORMAT);
allOptions.add(MANIFEST_TARGET_FILE_SIZE);
@@ -151,15 +151,15 @@ public class FileStoreOptions implements Serializable {
public static Path path(Map<String, String> options, ObjectIdentifier tableIdentifier) {
Preconditions.checkArgument(
- options.containsKey(FILE_PATH.key()),
+ options.containsKey(PATH.key()),
String.format(
"Failed to create file store path. "
+ "Please specify a root dir by setting session level configuration "
+ "as `SET 'table-store.%s' = '...'`. "
+ "Alternatively, you can use a per-table root dir "
+ "as `CREATE TABLE ${table} (...) WITH ('%s' = '...')`",
- FILE_PATH.key(), FILE_PATH.key()));
- return new Path(options.get(FILE_PATH.key()), relativeTablePath(tableIdentifier));
+ PATH.key(), PATH.key()));
+ return new Path(options.get(PATH.key()), relativeTablePath(tableIdentifier));
}
public static String relativeTablePath(ObjectIdentifier tableIdentifier) {
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/TestFileStore.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/TestFileStore.java
index 1c5bbbf..97ec62e 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/TestFileStore.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/TestFileStore.java
@@ -97,7 +97,7 @@ public class TestFileStore extends FileStoreImpl {
conf.set(FileStoreOptions.FILE_FORMAT, format);
conf.set(FileStoreOptions.MANIFEST_FORMAT, format);
- conf.set(FileStoreOptions.FILE_PATH, root);
+ conf.set(FileStoreOptions.PATH, root);
conf.set(FileStoreOptions.BUCKET, numBuckets);
return new TestFileStore(conf, partitionType, keyType, valueType, mergeFunction);
@@ -117,7 +117,7 @@ public class TestFileStore extends FileStoreImpl {
keyType,
valueType,
mergeFunction);
- this.root = conf.getString(FileStoreOptions.FILE_PATH);
+ this.root = conf.getString(FileStoreOptions.PATH);
this.keySerializer = new RowDataSerializer(keyType);
this.valueSerializer = new RowDataSerializer(valueType);
}
[flink-table-store] 10/15: [FLINK-27136] Disable rolling file when writing level 0 sst files
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit d1233255ed350e4435c5c360b94d3d0698edb6f7
Author: tsreaper <ts...@gmail.com>
AuthorDate: Fri Apr 8 19:19:01 2022 +0800
[FLINK-27136] Disable rolling file when writing level 0 sst files
This closes #83
---
.../apache/flink/table/store/file/mergetree/sst/SstFileWriter.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileWriter.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileWriter.java
index c72826a..270670b 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileWriter.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/mergetree/sst/SstFileWriter.java
@@ -134,7 +134,10 @@ public class SstFileWriter {
private long maxSequenceNumber;
private SstRollingFile(int level) {
- super(suggestedFileSize);
+ // each level 0 sst file is a sorted run,
+ // we must not write rolling files for level 0 ssts
+ // otherwise we cannot reduce the number of sorted runs when compacting
+ super(level == 0 ? Long.MAX_VALUE : suggestedFileSize);
this.level = level;
this.serializer = new KeyValueSerializer(keyType, valueType);
this.keySerializer = new RowDataSerializer(keyType);
[flink-table-store] 15/15: [FLINK-27123] Introduce filter predicate for 'LIKE' and fix filter pushdown bug
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 6c9f3d384669a8086a9275c28a41c9908f510fe8
Author: Jane Chan <55...@users.noreply.github.com>
AuthorDate: Thu Apr 14 14:31:17 2022 +0800
[FLINK-27123] Introduce filter predicate for 'LIKE' and fix filter pushdown bug
This closes #87
---
.../store/connector/source/TableStoreSource.java | 56 ++-
.../CompositePkAndMultiPartitionedTableITCase.java | 2 +
.../store/connector/ReadWriteTableITCase.java | 192 +++++++-
.../store/file/predicate/PredicateConverter.java | 100 ++++-
.../table/store/file/predicate/StartsWith.java | 56 +++
.../table/store/file/predicate/PredicateTest.java | 495 ++++++++++++++++++++-
6 files changed, 865 insertions(+), 36 deletions(-)
diff --git a/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/source/TableStoreSource.java b/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/source/TableStoreSource.java
index 6071c64..8234d9e 100644
--- a/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/source/TableStoreSource.java
+++ b/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/source/TableStoreSource.java
@@ -39,6 +39,8 @@ import org.apache.flink.table.expressions.TypeLiteralExpression;
import org.apache.flink.table.expressions.ValueLiteralExpression;
import org.apache.flink.table.factories.DynamicTableFactory;
import org.apache.flink.table.store.connector.TableStore;
+import org.apache.flink.table.store.file.predicate.And;
+import org.apache.flink.table.store.file.predicate.Predicate;
import org.apache.flink.table.store.file.predicate.PredicateConverter;
import org.apache.flink.table.store.log.LogSourceProvider;
import org.apache.flink.table.store.log.LogStoreTableFactory;
@@ -49,6 +51,7 @@ import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
import java.util.stream.Collectors;
import static org.apache.flink.table.store.connector.TableStoreFactoryOptions.SCAN_PARALLELISM;
@@ -71,8 +74,8 @@ public class TableStoreSource
private final DynamicTableFactory.Context logStoreContext;
@Nullable private final LogStoreTableFactory logStoreTableFactory;
- private List<ResolvedExpression> partitionFilters = new ArrayList<>();
- private List<ResolvedExpression> fieldFilters = new ArrayList<>();
+ @Nullable private Predicate partitionPredicate;
+ @Nullable private Predicate fieldPredicate;
@Nullable private int[][] projectFields;
public TableStoreSource(
@@ -142,8 +145,8 @@ public class TableStoreSource
.withContinuousMode(streaming)
.withLogSourceProvider(logSourceProvider)
.withProjection(projectFields)
- .withPartitionPredicate(PredicateConverter.convert(partitionFilters))
- .withFieldPredicate(PredicateConverter.convert(fieldFilters))
+ .withPartitionPredicate(partitionPredicate)
+ .withFieldPredicate(fieldPredicate)
.withParallelism(tableStore.options().get(SCAN_PARALLELISM));
return new DataStreamScanProvider() {
@@ -164,8 +167,8 @@ public class TableStoreSource
public DynamicTableSource copy() {
TableStoreSource copied =
new TableStoreSource(tableStore, streaming, logStoreContext, logStoreTableFactory);
- copied.partitionFilters = new ArrayList<>(partitionFilters);
- copied.fieldFilters = new ArrayList<>(fieldFilters);
+ copied.partitionPredicate = partitionPredicate;
+ copied.fieldPredicate = fieldPredicate;
copied.projectFields = projectFields;
return copied;
}
@@ -177,13 +180,29 @@ public class TableStoreSource
@Override
public Result applyFilters(List<ResolvedExpression> filters) {
- if (tableStore.partitioned()) {
- classifyFilters(filters);
- } else {
- fieldFilters = filters;
+ List<Predicate> partitionPredicates = new ArrayList<>();
+ List<Predicate> fieldPredicates = new ArrayList<>();
+ List<ResolvedExpression> notAcceptedFilters = new ArrayList<>();
+ for (ResolvedExpression filter : filters) {
+ Optional<ResolvedExpression> optionalPartPredicate = extractPartitionFilter(filter);
+ if (optionalPartPredicate.isPresent()) {
+ ResolvedExpression partitionFilter = optionalPartPredicate.get();
+ Optional<Predicate> partitionPredicate =
+ PredicateConverter.convert(partitionFilter);
+ if (partitionPredicate.isPresent()) {
+ partitionPredicates.add(partitionPredicate.get());
+ } else {
+ notAcceptedFilters.add(partitionFilter);
+ }
+ } else {
+ notAcceptedFilters.add(filter);
+ PredicateConverter.convert(filter).ifPresent(fieldPredicates::add);
+ }
}
+ partitionPredicate = partitionPredicates.stream().reduce(And::new).orElse(null);
+ fieldPredicate = fieldPredicates.stream().reduce(And::new).orElse(null);
return Result.of(
- filters, streaming && logStoreTableFactory != null ? filters : fieldFilters);
+ filters, streaming && logStoreTableFactory != null ? filters : notAcceptedFilters);
}
@Override
@@ -196,20 +215,17 @@ public class TableStoreSource
this.projectFields = projectedFields;
}
- private void classifyFilters(List<ResolvedExpression> filters) {
+ private Optional<ResolvedExpression> extractPartitionFilter(ResolvedExpression filter) {
List<String> fieldNames = tableStore.fieldNames();
List<String> partitionKeys = tableStore.partitionKeys();
PartitionIndexVisitor visitor =
new PartitionIndexVisitor(
fieldNames.stream().mapToInt(partitionKeys::indexOf).toArray());
- filters.forEach(
- filter -> {
- try {
- partitionFilters.add(filter.accept(visitor));
- } catch (FoundFieldReference e) {
- fieldFilters.add(filter);
- }
- });
+ try {
+ return Optional.of(filter.accept(visitor));
+ } catch (FoundFieldReference e) {
+ return Optional.empty();
+ }
}
private static class PartitionIndexVisitor implements ExpressionVisitor<ResolvedExpression> {
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
index a0a6b3f..8ae20b0 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/CompositePkAndMultiPartitionedTableITCase.java
@@ -24,6 +24,7 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.store.file.utils.BlockingIterator;
import org.apache.flink.types.Row;
+import org.junit.Ignore;
import org.junit.Test;
import java.util.ArrayList;
@@ -880,6 +881,7 @@ public class CompositePkAndMultiPartitionedTableITCase extends ReadWriteTableTes
changelogRow("+I", "US Dollar", "Yen")));
}
+ @Ignore
@Test
public void testEnableLogAndStreamingReadWriteMultiPartitionedRecordsWithoutPk()
throws Exception {
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
index f04fb5e..5d32712 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
@@ -35,6 +35,7 @@ import org.apache.flink.table.factories.DynamicTableFactory;
import org.apache.flink.table.factories.FactoryUtil;
import org.apache.flink.table.runtime.connector.sink.SinkRuntimeProviderContext;
import org.apache.flink.table.store.connector.sink.TableStoreSink;
+import org.apache.flink.table.store.file.FileStoreOptions;
import org.apache.flink.table.store.file.utils.BlockingIterator;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.RowType;
@@ -50,6 +51,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
@@ -110,7 +112,7 @@ public class ReadWriteTableITCase extends ReadWriteTableTestBase {
tEnv,
managedTable,
Collections.emptyMap(),
- "dt = '2022-01-02'",
+ "dt IN ('2022-01-02')",
Arrays.asList(
// part = 2022-01-01
changelogRow("+I", "Euro", 100L, "2022-01-02"),
@@ -127,7 +129,7 @@ public class ReadWriteTableITCase extends ReadWriteTableTestBase {
true, true, "dt <> '2022-01-02'", Collections.emptyList(), expected);
collectAndCheckBatchReadWrite(
- true, true, "dt = '2022-01-01'", Collections.emptyList(), expected);
+ true, true, "dt IN ('2022-01-01')", Collections.emptyList(), expected);
// test field filter
collectAndCheckBatchReadWrite(
@@ -211,7 +213,7 @@ public class ReadWriteTableITCase extends ReadWriteTableTestBase {
streamTableEnv,
managedTable,
Collections.emptyMap(),
- null,
+ "dt IN ('2022-01-01', '2022-01-02')",
Arrays.asList(
// part = 2022-01-01
changelogRow("+I", "Euro", 90L, "2022-01-01"),
@@ -323,7 +325,7 @@ public class ReadWriteTableITCase extends ReadWriteTableTestBase {
collectAndCheckBatchReadWrite(
false,
true,
- "currency = 'Yen'",
+ "currency IN ('Yen')",
Collections.singletonList("rate"),
Collections.singletonList(changelogRow("+I", 1L)));
}
@@ -1224,6 +1226,188 @@ public class ReadWriteTableITCase extends ReadWriteTableTestBase {
.containsOnly(changelogRow("+I", 1, "abc"));
}
+ @Test
+ public void testLike() throws Exception {
+ rootPath = TEMPORARY_FOLDER.newFolder().getPath();
+ tEnv = StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
+ List<Row> input =
+ Arrays.asList(
+ changelogRow("+I", 1, "test_1"),
+ changelogRow("+I", 2, "test_2"),
+ changelogRow("+I", 1, "test_%"),
+ changelogRow("+I", 2, "test%2"),
+ changelogRow("+I", 3, "university"),
+ changelogRow("+I", 4, "very"),
+ changelogRow("+I", 5, "yield"));
+ String id = registerData(input);
+ UUID randomSourceId = UUID.randomUUID();
+ tEnv.executeSql(
+ String.format(
+ "create table `helper_source_%s` (f0 int, f1 string) with ("
+ + "'connector' = 'values', "
+ + "'bounded' = 'true', "
+ + "'data-id' = '%s')",
+ randomSourceId, id));
+
+ UUID randomSinkId = UUID.randomUUID();
+ tEnv.executeSql(
+ String.format(
+ "create table `managed_table_%s` with ("
+ + "'%s' = '%s'"
+ + ") like `helper_source_%s` "
+ + "(excluding options)",
+ randomSinkId, FileStoreOptions.PATH.key(), rootPath, randomSourceId));
+
+ // insert multiple times
+ tEnv.executeSql(
+ String.format(
+ "insert into `managed_table_%s` select * from `helper_source_%s`",
+ randomSinkId, randomSourceId))
+ .await();
+
+ tEnv.executeSql(
+ String.format(
+ "insert into `managed_table_%s` values (7, 'villa'), (8, 'tests'), (20, 'test_123')",
+ randomSinkId))
+ .await();
+
+ tEnv.executeSql(
+ "insert into `managed_table_"
+ + randomSinkId
+ + "` values (9, 'valley'), (10, 'tested'), (100, 'test%fff')")
+ .await();
+
+ collectAndCheck(
+ tEnv,
+ "managed_table_" + randomSinkId,
+ Collections.emptyMap(),
+ "f1 like 'test%'",
+ Arrays.asList(
+ changelogRow("+I", 1, "test_1"),
+ changelogRow("+I", 2, "test_2"),
+ changelogRow("+I", 1, "test_%"),
+ changelogRow("+I", 2, "test%2"),
+ changelogRow("+I", 8, "tests"),
+ changelogRow("+I", 10, "tested"),
+ changelogRow("+I", 20, "test_123")));
+ collectAndCheck(
+ tEnv,
+ "managed_table_" + randomSinkId,
+ Collections.emptyMap(),
+ "f1 like 'v%'",
+ Arrays.asList(
+ changelogRow("+I", 4, "very"),
+ changelogRow("+I", 7, "villa"),
+ changelogRow("+I", 9, "valley")));
+ collectAndCheck(
+ tEnv,
+ "managed_table_" + randomSinkId,
+ Collections.emptyMap(),
+ "f1 like 'test=_%' escape '='",
+ Arrays.asList(
+ changelogRow("+I", 1, "test_1"),
+ changelogRow("+I", 2, "test_2"),
+ changelogRow("+I", 1, "test_%"),
+ changelogRow("+I", 20, "test_123")));
+ collectAndCheck(
+ tEnv,
+ "managed_table_" + randomSinkId,
+ Collections.emptyMap(),
+ "f1 like 'test=__' escape '='",
+ Arrays.asList(
+ changelogRow("+I", 1, "test_1"),
+ changelogRow("+I", 2, "test_2"),
+ changelogRow("+I", 1, "test_%")));
+ collectAndCheck(
+ tEnv,
+ "managed_table_" + randomSinkId,
+ Collections.emptyMap(),
+ "f1 like 'test$%%' escape '$'",
+ Arrays.asList(
+ changelogRow("+I", 2, "test%2"), changelogRow("+I", 100, "test%fff")));
+ }
+
+ @Test
+ public void testIn() throws Exception {
+ rootPath = TEMPORARY_FOLDER.newFolder().getPath();
+ tEnv = StreamTableEnvironment.create(buildBatchEnv(), EnvironmentSettings.inBatchMode());
+ List<Row> input =
+ Arrays.asList(
+ changelogRow("+I", 1, "aaa"),
+ changelogRow("+I", 2, "bbb"),
+ changelogRow("+I", 3, "ccc"),
+ changelogRow("+I", 4, "ddd"),
+ changelogRow("+I", 5, "eee"),
+ changelogRow("+I", 6, "aaa"),
+ changelogRow("+I", 7, "bbb"),
+ changelogRow("+I", 8, "ccc"),
+ changelogRow("+I", 9, "ddd"),
+ changelogRow("+I", 10, "eee"),
+ changelogRow("+I", 11, "aaa"),
+ changelogRow("+I", 12, "bbb"),
+ changelogRow("+I", 13, "ccc"),
+ changelogRow("+I", 14, "ddd"),
+ changelogRow("+I", 15, "eee"),
+ changelogRow("+I", 16, "aaa"),
+ changelogRow("+I", 17, "bbb"),
+ changelogRow("+I", 18, "ccc"),
+ changelogRow("+I", 19, "ddd"),
+ changelogRow("+I", 20, "eee"),
+ changelogRow("+I", 21, "fff"));
+
+ String id = registerData(input);
+ UUID randomSourceId = UUID.randomUUID();
+ tEnv.executeSql(
+ String.format(
+ "create table `helper_source_%s` (f0 int, f1 string) with ("
+ + "'connector' = 'values', "
+ + "'bounded' = 'true', "
+ + "'data-id' = '%s')",
+ randomSourceId, id));
+
+ UUID randomSinkId = UUID.randomUUID();
+ tEnv.executeSql(
+ String.format(
+ "create table `managed_table_%s` with ("
+ + "'%s' = '%s'"
+ + ") like `helper_source_%s` "
+ + "(excluding options)",
+ randomSinkId, FileStoreOptions.PATH.key(), rootPath, randomSourceId));
+ tEnv.executeSql(
+ String.format(
+ "insert into `managed_table_%s` select * from `helper_source_%s`",
+ randomSinkId, randomSourceId))
+ .await();
+
+ collectAndCheck(
+ tEnv,
+ "managed_table_" + randomSinkId,
+ Collections.emptyMap(),
+ "f0 in (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, "
+ + "11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21)",
+ input);
+
+ collectAndCheck(
+ tEnv,
+ "managed_table_" + randomSinkId,
+ Collections.emptyMap(),
+ "f1 in ('aaa', 'bbb', 'ccc', 'ddd', 'eee', 'fff')",
+ input);
+ }
+
+ @Test
+ public void testUnsupportedPredicate() throws Exception {
+ // test unsupported filter
+ collectAndCheckBatchReadWrite(
+ true,
+ true,
+ "currency similar to 'Euro'",
+ Collections.emptyList(),
+ Arrays.asList(
+ changelogRow("+I", "Euro", 114L, "2022-01-01"),
+ changelogRow("+I", "Euro", 119L, "2022-01-02")));
+ }
+
// ------------------------ Tools ----------------------------------
private String collectAndCheckBatchReadWrite(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/predicate/PredicateConverter.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/predicate/PredicateConverter.java
index f845dec..a01b261 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/predicate/PredicateConverter.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/predicate/PredicateConverter.java
@@ -18,6 +18,7 @@
package org.apache.flink.table.store.file.predicate;
+import org.apache.flink.table.data.binary.BinaryStringData;
import org.apache.flink.table.expressions.CallExpression;
import org.apache.flink.table.expressions.Expression;
import org.apache.flink.table.expressions.ExpressionVisitor;
@@ -27,10 +28,13 @@ import org.apache.flink.table.expressions.TypeLiteralExpression;
import org.apache.flink.table.expressions.ValueLiteralExpression;
import org.apache.flink.table.functions.BuiltInFunctionDefinitions;
import org.apache.flink.table.functions.FunctionDefinition;
+import org.apache.flink.table.functions.SqlLikeUtils;
import org.apache.flink.table.store.utils.TypeUtils;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeFamily;
import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.VarCharType;
import javax.annotation.Nullable;
@@ -38,6 +42,8 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BiFunction;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import static org.apache.flink.table.data.conversion.DataStructureConverters.getConverter;
import static org.apache.flink.table.types.logical.utils.LogicalTypeCasts.supportsImplicitCast;
@@ -47,6 +53,9 @@ public class PredicateConverter implements ExpressionVisitor<Predicate> {
public static final PredicateConverter CONVERTER = new PredicateConverter();
+ /** Accepts simple LIKE patterns like "abc%". */
+ private static final Pattern BEGIN_PATTERN = Pattern.compile("([^%]+)%");
+
@Nullable
public Predicate fromMap(Map<String, String> map, RowType rowType) {
List<String> fieldNames = rowType.getFieldNames();
@@ -95,9 +104,78 @@ public class PredicateConverter implements ExpressionVisitor<Predicate> {
.map(FieldReferenceExpression::getFieldIndex)
.map(IsNotNull::new)
.orElseThrow(UnsupportedExpression::new);
+ } else if (func == BuiltInFunctionDefinitions.LIKE) {
+ FieldReferenceExpression fieldRefExpr =
+ extractFieldReference(children.get(0)).orElseThrow(UnsupportedExpression::new);
+ if (fieldRefExpr
+ .getOutputDataType()
+ .getLogicalType()
+ .is(LogicalTypeFamily.CHARACTER_STRING)) {
+ String sqlPattern =
+ extractLiteral(fieldRefExpr.getOutputDataType(), children.get(1))
+ .orElseThrow(UnsupportedExpression::new)
+ .value()
+ .toString();
+ String escape =
+ children.size() <= 2
+ ? null
+ : extractLiteral(fieldRefExpr.getOutputDataType(), children.get(2))
+ .orElseThrow(UnsupportedExpression::new)
+ .value()
+ .toString();
+ String escapedSqlPattern = sqlPattern;
+ boolean allowQuick = false;
+ if (escape == null && !sqlPattern.contains("_")) {
+ allowQuick = true;
+ } else if (escape != null) {
+ if (escape.length() != 1) {
+ throw SqlLikeUtils.invalidEscapeCharacter(escape);
+ }
+ char escapeChar = escape.charAt(0);
+ boolean matched = true;
+ int i = 0;
+ StringBuilder sb = new StringBuilder();
+ while (i < sqlPattern.length() && matched) {
+ char c = sqlPattern.charAt(i);
+ if (c == escapeChar) {
+ if (i == (sqlPattern.length() - 1)) {
+ throw SqlLikeUtils.invalidEscapeSequence(sqlPattern, i);
+ }
+ char nextChar = sqlPattern.charAt(i + 1);
+ if (nextChar == '%') {
+ matched = false;
+ } else if ((nextChar == '_') || (nextChar == escapeChar)) {
+ sb.append(nextChar);
+ i += 1;
+ } else {
+ throw SqlLikeUtils.invalidEscapeSequence(sqlPattern, i);
+ }
+ } else if (c == '_') {
+ matched = false;
+ } else {
+ sb.append(c);
+ }
+ i = i + 1;
+ }
+ if (matched) {
+ allowQuick = true;
+ escapedSqlPattern = sb.toString();
+ }
+ }
+ if (allowQuick) {
+ Matcher beginMatcher = BEGIN_PATTERN.matcher(escapedSqlPattern);
+ if (beginMatcher.matches()) {
+ return new StartsWith(
+ fieldRefExpr.getFieldIndex(),
+ new Literal(
+ VarCharType.STRING_TYPE,
+ BinaryStringData.fromString(beginMatcher.group(1))));
+ }
+ }
+ }
}
- // TODO is_xxx, between_xxx, like, similar, in, not_in, not?
+ // TODO is_xxx, between_xxx, similar, in, not_in, not?
throw new UnsupportedExpression();
}
@@ -213,14 +291,18 @@ public class PredicateConverter implements ExpressionVisitor<Predicate> {
throw new RuntimeException("Unsupported expression: " + expression.asSummaryString());
}
- @Nullable
- public static Predicate convert(List<ResolvedExpression> filters) {
- return filters != null
- ? filters.stream()
- .map(filter -> filter.accept(PredicateConverter.CONVERTER))
- .reduce(And::new)
- .orElse(null)
- : null;
+ /**
+ * Try best to convert a {@link ResolvedExpression} to {@link Predicate}.
+ *
+ * @param filter a resolved expression
+ * @return {@link Predicate} if no {@link UnsupportedExpression} thrown.
+ */
+ public static Optional<Predicate> convert(ResolvedExpression filter) {
+ try {
+ return Optional.ofNullable(filter.accept(PredicateConverter.CONVERTER));
+ } catch (UnsupportedExpression e) {
+ return Optional.empty();
+ }
}
/** Encounter an unsupported expression, the caller can choose to ignore this filter branch. */
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/predicate/StartsWith.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/predicate/StartsWith.java
new file mode 100644
index 0000000..7e2d9b6
--- /dev/null
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/predicate/StartsWith.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.predicate;
+
+import org.apache.flink.table.data.binary.BinaryStringData;
+import org.apache.flink.table.store.file.stats.FieldStats;
+
+/** A {@link Predicate} to evaluate {@code filter like 'abc%' or filter like 'abc_'}. */
+public class StartsWith implements Predicate {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int index;
+
+ private final Literal patternLiteral;
+
+ public StartsWith(int index, Literal patternLiteral) {
+ this.index = index;
+ this.patternLiteral = patternLiteral;
+ }
+
+ @Override
+ public boolean test(Object[] values) {
+ BinaryStringData field = (BinaryStringData) values[index];
+ return field != null && field.startsWith((BinaryStringData) patternLiteral.value());
+ }
+
+ @Override
+ public boolean test(long rowCount, FieldStats[] fieldStats) {
+ FieldStats stats = fieldStats[index];
+ if (rowCount == stats.nullCount()) {
+ return false;
+ }
+ BinaryStringData min = (BinaryStringData) stats.minValue();
+ BinaryStringData max = (BinaryStringData) stats.maxValue();
+ BinaryStringData pattern = (BinaryStringData) patternLiteral.value();
+ return (min.startsWith(pattern) || min.compareTo(pattern) <= 0)
+ && (max.startsWith(pattern) || max.compareTo(pattern) >= 0);
+ }
+}
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/predicate/PredicateTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/predicate/PredicateTest.java
index 5b9d3f5..4b4a25f 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/predicate/PredicateTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/predicate/PredicateTest.java
@@ -19,6 +19,7 @@
package org.apache.flink.table.store.file.predicate;
import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.StringData;
import org.apache.flink.table.expressions.CallExpression;
import org.apache.flink.table.expressions.FieldReferenceExpression;
import org.apache.flink.table.expressions.ResolvedExpression;
@@ -50,8 +51,12 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
+import static org.apache.flink.table.api.DataTypes.STRING;
import static org.apache.flink.table.planner.expressions.ExpressionBuilder.literal;
import static org.apache.flink.table.store.file.predicate.PredicateConverter.CONVERTER;
import static org.assertj.core.api.Assertions.assertThat;
@@ -315,6 +320,28 @@ public class PredicateTest {
.isEqualTo(false);
}
+ @MethodSource("provideLikeExpressions")
+ @ParameterizedTest
+ public void testStartsWith(
+ CallExpression callExpression,
+ List<Object[]> valuesList,
+ List<Boolean> expectedForValues,
+ List<Long> rowCountList,
+ List<FieldStats[]> statsList,
+ List<Boolean> expectedForStats) {
+ Predicate predicate = callExpression.accept(CONVERTER);
+ IntStream.range(0, valuesList.size())
+ .forEach(
+ i ->
+ assertThat(predicate.test(valuesList.get(i)))
+ .isEqualTo(expectedForValues.get(i)));
+ IntStream.range(0, rowCountList.size())
+ .forEach(
+ i ->
+ assertThat(predicate.test(rowCountList.get(i), statsList.get(i)))
+ .isEqualTo(expectedForStats.get(i)));
+ }
+
@Test
public void testUnsupportedExpression() {
CallExpression expression =
@@ -325,13 +352,337 @@ public class PredicateTest {
field(0, DataTypes.INT()),
literal(3)),
call(
- BuiltInFunctionDefinitions.LIKE,
+ BuiltInFunctionDefinitions.SIMILAR,
field(1, DataTypes.INT()),
literal(5)));
assertThatThrownBy(() -> expression.accept(CONVERTER))
.isInstanceOf(PredicateConverter.UnsupportedExpression.class);
}
+ @Test
+ public void testUnsupportedStartsPatternForLike() {
+ // starts pattern with '_' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("abc_", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // starts pattern like 'abc%xyz' or 'abc_xyz'
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("abc%xyz", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("abc_xyz", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // starts pattern like 'abc%xyz' or 'abc_xyz' with '%' or '_' to escape
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "=%abc=%%xyz=_",
+ STRING()), // matches "%abc%(?s:.*)xyz_"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "abc=%%xyz",
+ STRING()), // matches "abc%(?s:.*)xyz"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "abc=%_xyz",
+ STRING()), // matches "abc%.xyz"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "abc=_%xyz",
+ STRING()), // matches "abc_(?s:.*)xyz"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "abc=__xyz",
+ STRING()), // matches "abc_.xyz"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // starts pattern with wildcard '%' at the beginning to escape
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("=%%", STRING()), // matches "%(?s:.*)"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ }
+
+ @Test
+ public void testUnsupportedEndsPatternForLike() {
+ // ends pattern with '%' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("%456", STRING())) // matches "(?s:.*)456"
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // ends pattern with '_' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("_456", STRING())) // matches ".456"
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // ends pattern with '[]' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("_[456]", STRING())) // matches ".[456]"
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "%[h-m]",
+ STRING())) // matches "(?s:.*)[h-m]"
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // ends pattern with '[^]' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "%[^h-m]",
+ STRING())) // matches "(?s:.*)[^h-m]"
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("_[^xyz]", STRING())) // matches ".[^xyz]"
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // ends pattern escape wildcard '%'
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "%=%456",
+ STRING()), // matches "(?s:.*)%456"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "%=_456",
+ STRING()), // matches "(?s:.*)_456"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // ends pattern escape wildcard '_'
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("_=_456", STRING()), // matches "._456"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ }
+
+ @Test
+ public void testUnsupportedEqualsPatternForLike() {
+ // equals pattern
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("123456", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // equals pattern escape '%'
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("12=%45", STRING()), // equals "12%45"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // equals pattern escape '_'
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("12=_45", STRING()), // equals "12_45"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ }
+
+ @Test
+ public void testUnsupportedMiddlePatternForLike() {
+ // middle pattern with '%' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("%345%", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // middle pattern with '_' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("_345_", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // middle pattern with both '%' and '_' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("_345%", STRING())) // matches ".345(?s:.*)"
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("%345_", STRING())) // matches "(?s:.*)345."
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // middle pattern with '[]' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "%[a-c]_",
+ STRING())) // matches "(?s:.*)[a-c]."
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // middle pattern with '[^]' as wildcard
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "%[^abc]_",
+ STRING())) // matches "(?s:.*)[^abc]."
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // middle pattern escape '%'
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "%34=%5%",
+ STRING()), // matches "(?s:.*)34%5(.*)"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+
+ // middle pattern escape '_'
+ assertThatThrownBy(
+ () ->
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal(
+ "%34=_5%",
+ STRING()), // matches "(?s:.*)34_5(.*)"
+ literal("=", STRING()))
+ .accept(CONVERTER))
+ .isInstanceOf(PredicateConverter.UnsupportedExpression.class);
+ }
+
@Test
public void testUnsupportedType() {
DataType structType = DataTypes.ROW(DataTypes.INT()).bridgedTo(Row.class);
@@ -378,6 +729,144 @@ public class PredicateTest {
TypeUtils.castFromString("2022-03-25 15:00:02", timestampType)));
}
+ public static Stream<Arguments> provideLikeExpressions() {
+ CallExpression expr1 =
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("abd%", STRING()));
+ List<Object[]> valuesList1 =
+ Arrays.asList(
+ new Object[] {null},
+ new Object[] {StringData.fromString("a")},
+ new Object[] {StringData.fromString("ab")},
+ new Object[] {StringData.fromString("abd")},
+ new Object[] {StringData.fromString("abd%")},
+ new Object[] {StringData.fromString("abd1")},
+ new Object[] {StringData.fromString("abde@")},
+ new Object[] {StringData.fromString("abd_")},
+ new Object[] {StringData.fromString("abd_%")});
+ List<Boolean> expectedForValues1 =
+ Arrays.asList(false, false, false, true, true, true, true, true, true);
+ List<Long> rowCountList1 = Arrays.asList(0L, 3L, 3L, 3L);
+ List<FieldStats[]> statsList1 =
+ Arrays.asList(
+ new FieldStats[] {new FieldStats(null, null, 0L)},
+ new FieldStats[] {new FieldStats(null, null, 3L)},
+ new FieldStats[] {
+ new FieldStats(
+ StringData.fromString("ab"),
+ StringData.fromString("abc123"),
+ 1L)
+ },
+ new FieldStats[] {
+ new FieldStats(
+ StringData.fromString("abc"), StringData.fromString("abe"), 1L)
+ });
+ List<Boolean> expectedForStats1 = Arrays.asList(false, false, false, true);
+
+ CallExpression expr2 =
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("test=_%", STRING()),
+ literal("=", STRING()));
+ List<Object[]> valuesList2 =
+ Arrays.asList(
+ new Object[] {StringData.fromString("test%")},
+ new Object[] {StringData.fromString("test_123")},
+ new Object[] {StringData.fromString("test_%")},
+ new Object[] {StringData.fromString("test__")});
+ List<Boolean> expectedForValues2 = Arrays.asList(false, true, true, true);
+ List<Long> rowCountList2 = Collections.singletonList(3L);
+ List<FieldStats[]> statsList2 =
+ Collections.singletonList(
+ new FieldStats[] {
+ new FieldStats(
+ StringData.fromString("test_123"),
+ StringData.fromString("test_789"),
+ 0L)
+ });
+ List<Boolean> expectedForStats2 = Collections.singletonList(true);
+
+ // currently, SQL wildcards '[]' and '[^]' are deemed as normal characters in Flink
+ CallExpression expr3 =
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("[a-c]xyz%", STRING()));
+ List<Object[]> valuesList3 =
+ Arrays.asList(
+ new Object[] {StringData.fromString("axyz")},
+ new Object[] {StringData.fromString("bxyz")},
+ new Object[] {StringData.fromString("cxyz")},
+ new Object[] {StringData.fromString("[a-c]xyz")});
+ List<Boolean> expectedForValues3 = Arrays.asList(false, false, false, true);
+ List<Long> rowCountList3 = Collections.singletonList(3L);
+ List<FieldStats[]> statsList3 =
+ Collections.singletonList(
+ new FieldStats[] {
+ new FieldStats(
+ StringData.fromString("[a-c]xyz"),
+ StringData.fromString("[a-c]xyzz"),
+ 0L)
+ });
+ List<Boolean> expectedForStats3 = Collections.singletonList(true);
+
+ CallExpression expr4 =
+ call(
+ BuiltInFunctionDefinitions.LIKE,
+ field(0, STRING()),
+ literal("[^a-d]xyz%", STRING()));
+ List<Object[]> valuesList4 =
+ Arrays.asList(
+ new Object[] {StringData.fromString("exyz")},
+ new Object[] {StringData.fromString("fxyz")},
+ new Object[] {StringData.fromString("axyz")},
+ new Object[] {StringData.fromString("[^a-d]xyz")});
+ List<Boolean> expectedForValues4 = Arrays.asList(false, false, false, true);
+ List<Long> rowCountList4 = Collections.singletonList(3L);
+ List<FieldStats[]> statsList4 =
+ Collections.singletonList(
+ new FieldStats[] {
+ new FieldStats(
+ StringData.fromString("[^a-d]xyz"),
+ StringData.fromString("[^a-d]xyzz"),
+ 1L)
+ });
+ List<Boolean> expectedForStats4 = Collections.singletonList(true);
+
+ return Stream.of(
+ Arguments.of(
+ expr1,
+ valuesList1,
+ expectedForValues1,
+ rowCountList1,
+ statsList1,
+ expectedForStats1),
+ Arguments.of(
+ expr2,
+ valuesList2,
+ expectedForValues2,
+ rowCountList2,
+ statsList2,
+ expectedForStats2),
+ Arguments.of(
+ expr3,
+ valuesList3,
+ expectedForValues3,
+ rowCountList3,
+ statsList3,
+ expectedForStats3),
+ Arguments.of(
+ expr4,
+ valuesList4,
+ expectedForValues4,
+ rowCountList4,
+ statsList4,
+ expectedForStats4));
+ }
+
private byte[] writeObject(Literal literal) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
ObjectOutputStream oos = new ObjectOutputStream(baos);
@@ -394,11 +883,11 @@ public class PredicateTest {
return object;
}
- private FieldReferenceExpression field(int i, DataType type) {
+ private static FieldReferenceExpression field(int i, DataType type) {
return new FieldReferenceExpression("name", type, 0, i);
}
- private CallExpression call(FunctionDefinition function, ResolvedExpression... args) {
+ private static CallExpression call(FunctionDefinition function, ResolvedExpression... args) {
return new CallExpression(function, Arrays.asList(args), DataTypes.BOOLEAN());
}
}
[flink-table-store] 04/15: [FLINK-26913] Add IT case for computed column and watermark
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit feb11f160028a553a474ef6516e9fba7519394cb
Author: Jane Chan <55...@users.noreply.github.com>
AuthorDate: Fri Apr 8 09:41:14 2022 +0800
[FLINK-26913] Add IT case for computed column and watermark
This closes #76
---
.../ComputedColumnAndWatermarkTableITCase.java | 261 +++++++++++++++++++++
.../store/connector/ReadWriteTableITCase.java | 4 +-
.../store/connector/ReadWriteTableTestBase.java | 81 ++++++-
.../store/connector/ReadWriteTableTestUtil.java | 66 +++++-
.../table/store/connector/ShowCreateUtil.java | 11 +-
5 files changed, 403 insertions(+), 20 deletions(-)
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ComputedColumnAndWatermarkTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ComputedColumnAndWatermarkTableITCase.java
new file mode 100644
index 0000000..a7fe99b
--- /dev/null
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ComputedColumnAndWatermarkTableITCase.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.connector;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.table.store.log.LogOptions;
+
+import org.junit.Test;
+
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.rates;
+import static org.apache.flink.table.store.connector.ReadWriteTableTestUtil.ratesWithTimestamp;
+import static org.apache.flink.table.store.log.LogOptions.LOG_PREFIX;
+
+/** Table store IT case when the managed table has computed column and watermark spec. */
+public class ComputedColumnAndWatermarkTableITCase extends ReadWriteTableTestBase {
+
+ @Test
+ public void testBatchSelectComputedColumn() throws Exception {
+ // input is rates()
+ collectAndCheckUnderSameEnv(
+ false,
+ false,
+ true,
+ Collections.emptyList(), // partition
+ Collections.emptyList(), // pk
+ Collections.singletonList(Tuple2.of("capital_currency", "UPPER(currency)")),
+ null,
+ true,
+ Collections.emptyMap(),
+ null,
+ Collections.singletonList("capital_currency"),
+ rates().stream()
+ .map(
+ row ->
+ changelogRow(
+ row.getKind().shortString(),
+ ((String) row.getField(0)).toUpperCase()))
+ .collect(Collectors.toList()));
+
+ // input is rates()
+ collectAndCheckUnderSameEnv(
+ false,
+ false,
+ true,
+ Collections.emptyList(), // partition
+ Collections.singletonList("currency"), // pk
+ Collections.singletonList(Tuple2.of("capital_currency", "LOWER(currency)")),
+ null,
+ true,
+ Collections.emptyMap(),
+ null,
+ Collections.singletonList("capital_currency"),
+ Arrays.asList(
+ changelogRow("+I", "us dollar"),
+ changelogRow("+I", "yen"),
+ changelogRow("+I", "euro")));
+
+ // input is hourlyRates()
+ collectAndCheckUnderSameEnv(
+ false,
+ false,
+ true,
+ Arrays.asList("dt", "hh"), // partition
+ Arrays.asList("currency", "dt", "hh"), // pk
+ Collections.singletonList(Tuple2.of("dth", "dt || ' ' || hh")),
+ null,
+ true,
+ Collections.emptyMap(),
+ "dth = '2022-01-02 12'",
+ Collections.singletonList("dth"),
+ Collections.singletonList(changelogRow("+I", "2022-01-02 12")));
+
+ // test proctime
+ collectAndCheckUnderSameEnv(
+ false,
+ false,
+ true,
+ Collections.emptyList(), // partition
+ Collections.singletonList("currency"), // pk
+ Collections.singletonList(Tuple2.of("ptime", "PROCTIME()")),
+ null,
+ true,
+ Collections.emptyMap(),
+ "currency = 'US Dollar'",
+ Collections.singletonList("CHAR_LENGTH(DATE_FORMAT(ptime, 'yyyy-MM-dd HH:mm'))"),
+ Collections.singletonList(changelogRow("+I", 16)));
+ }
+
+ @Test
+ public void testStreamingSelectComputedColumn() throws Exception {
+ // input is ratesChangelogWithUB()
+ collectAndCheckUnderSameEnv(
+ true,
+ true,
+ false,
+ Collections.emptyList(), // partition
+ Collections.emptyList(), // pk
+ Arrays.asList(
+ Tuple2.of("capital_currency", "UPPER(currency)"),
+ Tuple2.of("ptime", "PROCTIME()")),
+ null,
+ true,
+ Collections.emptyMap(),
+ "currency IS NULL",
+ Arrays.asList(
+ "capital_currency",
+ "CHAR_LENGTH(DATE_FORMAT(ptime, 'yyyy-MM-dd HH:mm'))"),
+ Collections.singletonList(changelogRow("+I", null, 16)))
+ .f1
+ .close();
+
+ // input is dailyExchangeRatesChangelogWithoutUB()
+ collectAndCheckUnderSameEnv(
+ true,
+ true,
+ false,
+ Collections.singletonList("dt"), // partition
+ Arrays.asList("from_currency", "to_currency", "dt"), // pk
+ Arrays.asList(
+ Tuple2.of(
+ "corrected_rate_by_to_currency",
+ "COALESCE(rate_by_to_currency, 1)"),
+ Tuple2.of("ptime", "PROCTIME()")),
+ null,
+ false,
+ Collections.singletonMap(
+ LOG_PREFIX + LogOptions.SCAN.key(),
+ LogOptions.LogStartupMode.LATEST.name().toLowerCase()),
+ "rate_by_to_currency IS NULL",
+ Arrays.asList(
+ "corrected_rate_by_to_currency",
+ "CHAR_LENGTH(DATE_FORMAT(ptime, 'yyyy-MM-dd HH:mm'))"),
+ Collections.singletonList(changelogRow("+I", 1d, 16)))
+ .f1
+ .close();
+ }
+
+ @Test
+ public void testBatchSelectWithWatermark() throws Exception {
+ // input is ratesWithTimestamp(), test `ts` as an ordinary field under batch mode
+ collectAndCheckUnderSameEnv(
+ false,
+ false,
+ true,
+ Collections.emptyList(), // partition
+ Collections.emptyList(), // pk
+ Collections.emptyList(), // computed column
+ WatermarkSpec.of("ts", "ts - INTERVAL '3' YEAR"),
+ true,
+ Collections.emptyMap(), // read hints
+ null,
+ Collections.emptyList(), // projection
+ ratesWithTimestamp());
+ }
+
+ @Test
+ public void testStreamingSelectWithWatermark() throws Exception {
+ String lateEventFilter = "CURRENT_WATERMARK(ts) IS NULL OR ts > CURRENT_WATERMARK(ts)";
+ // input is ratesWithTimestamp()
+
+ // physical column as watermark
+ collectAndCheckUnderSameEnv(
+ true,
+ true,
+ true,
+ Collections.emptyList(), // partition
+ Collections.emptyList(), // pk
+ Collections.emptyList(), // computed column
+ WatermarkSpec.of("ts", "ts - INTERVAL '3' YEAR"),
+ false,
+ Collections.singletonMap(
+ LOG_PREFIX + LogOptions.SCAN.key(),
+ LogOptions.LogStartupMode.LATEST.name().toLowerCase()),
+ lateEventFilter,
+ Collections.emptyList(), // projection
+ Collections.singletonList(
+ changelogRow(
+ "+I",
+ "US Dollar",
+ 102L,
+ LocalDateTime.parse("1990-04-07T10:00:11.120"))))
+ .f1
+ .close();
+
+ // computed column as watermark
+ collectAndCheckUnderSameEnv(
+ true,
+ true,
+ true,
+ Collections.emptyList(), // partition
+ Collections.emptyList(), // pk
+ Collections.singletonList(Tuple2.of("ts1", "ts")), // computed column
+ WatermarkSpec.of("ts1", "ts1 - INTERVAL '3' YEAR"),
+ false,
+ Collections.singletonMap(
+ LOG_PREFIX + LogOptions.SCAN.key(),
+ LogOptions.LogStartupMode.LATEST.name().toLowerCase()),
+ lateEventFilter.replaceAll("ts", "ts1"),
+ Arrays.asList("currency", "rate", "ts1"),
+ Collections.singletonList(
+ changelogRow(
+ "+I",
+ "US Dollar",
+ 102L,
+ LocalDateTime.parse("1990-04-07T10:00:11.120"))))
+ .f1
+ .close();
+
+ // query both event time and processing time
+ collectAndCheckUnderSameEnv(
+ true,
+ true,
+ true,
+ Collections.emptyList(), // partition
+ Collections.emptyList(), // pk
+ Collections.singletonList(
+ Tuple2.of("ptime", "PROCTIME()")), // computed column
+ WatermarkSpec.of("ts", "ts - INTERVAL '3' YEAR"),
+ false,
+ Collections.singletonMap(
+ LOG_PREFIX + LogOptions.SCAN.key(),
+ LogOptions.LogStartupMode.LATEST.name().toLowerCase()),
+ lateEventFilter,
+ Arrays.asList(
+ "currency",
+ "rate",
+ "ts",
+ "CHAR_LENGTH(DATE_FORMAT(ptime, 'yyyy-MM-dd HH:mm'))"), // projection
+ Collections.singletonList(
+ changelogRow(
+ "+I",
+ "US Dollar",
+ 102L,
+ LocalDateTime.parse("1990-04-07T10:00:11.120"),
+ 16)))
+ .f1
+ .close();
+ }
+}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
index 535f097..4596dec 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableITCase.java
@@ -1175,7 +1175,9 @@ public class ReadWriteTableITCase extends ReadWriteTableTestBase {
false,
false,
Collections.emptyList(),
- Collections.emptyList())
+ Collections.emptyList(),
+ Collections.emptyList(),
+ null)
.f1;
// without hint
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
index 650166a..ad54b50 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestBase.java
@@ -22,11 +22,14 @@ import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.EnvironmentSettings;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.TableDescriptor;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.catalog.CatalogBaseTable;
import org.apache.flink.table.catalog.ObjectIdentifier;
import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
import org.apache.flink.table.store.file.FileStoreOptions;
import org.apache.flink.table.store.file.utils.BlockingIterator;
import org.apache.flink.table.store.kafka.KafkaTableTestBase;
@@ -157,6 +160,8 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
true,
partitions,
primaryKeys,
+ Collections.emptyList(),
+ null,
true,
Collections.emptyMap(),
filter,
@@ -181,6 +186,8 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
false,
partitions,
primaryKeys,
+ Collections.emptyList(),
+ null,
true,
readHints,
filter,
@@ -205,6 +212,8 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
false,
partitions,
primaryKeys,
+ Collections.emptyList(),
+ null,
true,
readHints,
filter,
@@ -230,6 +239,8 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
insertOnly,
partitionKeys,
primaryKeys,
+ Collections.emptyList(),
+ null,
false,
hints,
filter,
@@ -255,6 +266,8 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
insertOnly,
partitionKeys,
primaryKeys,
+ Collections.emptyList(),
+ null,
true,
hints,
null,
@@ -269,7 +282,9 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
boolean enableLogStore,
boolean insertOnly,
List<String> partitionKeys,
- List<String> primaryKeys)
+ List<String> primaryKeys,
+ List<Tuple2<String, String>> computedColumnExpressions,
+ @Nullable WatermarkSpec watermarkSpec)
throws Exception {
Map<String, String> tableOptions = new HashMap<>();
rootPath = TEMPORARY_FOLDER.newFolder().getPath();
@@ -286,23 +301,50 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
helperTableDdl =
insertOnly
? prepareHelperSourceWithInsertOnlyRecords(
- sourceTable, partitionKeys, primaryKeys)
+ sourceTable, partitionKeys, primaryKeys, watermarkSpec != null)
: prepareHelperSourceWithChangelogRecords(
- sourceTable, partitionKeys, primaryKeys);
+ sourceTable, partitionKeys, primaryKeys, watermarkSpec != null);
env = buildStreamEnv();
builder.inStreamingMode();
} else {
helperTableDdl =
prepareHelperSourceWithInsertOnlyRecords(
- sourceTable, partitionKeys, primaryKeys);
+ sourceTable, partitionKeys, primaryKeys, watermarkSpec != null);
env = buildBatchEnv();
builder.inBatchMode();
}
- String managedTableDdl = createTableLikeDDL(sourceTable, managedTable, tableOptions);
-
tEnv = StreamTableEnvironment.create(env, builder.build());
tEnv.executeSql(helperTableDdl);
- tEnv.executeSql(managedTableDdl);
+
+ String managedTableDdl;
+ if (computedColumnExpressions.isEmpty()) {
+ managedTableDdl =
+ createTableLikeDDL(sourceTable, managedTable, tableOptions, watermarkSpec);
+ tEnv.executeSql(managedTableDdl);
+ } else {
+ String cat = tEnv.getCurrentCatalog();
+ String db = tEnv.getCurrentDatabase();
+ ObjectPath objectPath = new ObjectPath(db, sourceTable);
+ ResolvedCatalogTable helperSource =
+ (ResolvedCatalogTable) tEnv.getCatalog(cat).get().getTable(objectPath);
+ Schema.Builder schemaBuilder =
+ Schema.newBuilder().fromSchema(helperSource.getUnresolvedSchema());
+ computedColumnExpressions.forEach(
+ tuple -> schemaBuilder.columnByExpression(tuple.f0, tuple.f1));
+
+ if (watermarkSpec != null) {
+ schemaBuilder.watermark(watermarkSpec.columnName, watermarkSpec.expressionAsString);
+ }
+
+ TableDescriptor.Builder descriptorBuilder =
+ TableDescriptor.forManaged()
+ .partitionedBy(helperSource.getPartitionKeys().toArray(new String[0]))
+ .schema(schemaBuilder.build());
+ tableOptions.forEach(descriptorBuilder::option);
+ tEnv.createTable(
+ ObjectIdentifier.of(cat, db, managedTable).asSerializableString(),
+ descriptorBuilder.build());
+ }
return new Tuple2<>(sourceTable, managedTable);
}
@@ -312,6 +354,8 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
boolean insertOnly,
List<String> partitionKeys,
List<String> primaryKeys,
+ List<Tuple2<String, String>> computedColumnExpressions,
+ @Nullable WatermarkSpec watermarkSpec,
boolean writeFirst,
Map<String, String> readHints,
@Nullable String filter,
@@ -320,7 +364,13 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
throws Exception {
Tuple2<String, String> tables =
createSourceAndManagedTable(
- streaming, enableLogStore, insertOnly, partitionKeys, primaryKeys);
+ streaming,
+ enableLogStore,
+ insertOnly,
+ partitionKeys,
+ primaryKeys,
+ computedColumnExpressions,
+ watermarkSpec);
String sourceTable = tables.f0;
String managedTable = tables.f1;
@@ -384,4 +434,19 @@ public class ReadWriteTableTestBase extends KafkaTableTestBase {
env.setParallelism(2);
return env;
}
+
+ /** A POJO class to help assign watermark by string. */
+ protected static class WatermarkSpec {
+ String columnName;
+ String expressionAsString;
+
+ private WatermarkSpec(String columnName, String expressionAsString) {
+ this.columnName = columnName;
+ this.expressionAsString = expressionAsString;
+ }
+
+ protected static WatermarkSpec of(String columnName, String expressionAsString) {
+ return new WatermarkSpec(columnName, expressionAsString);
+ }
+ }
}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestUtil.java
index 6c86a8f..3e1b813 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ReadWriteTableTestUtil.java
@@ -26,8 +26,10 @@ import org.apache.flink.table.catalog.ResolvedCatalogTable;
import org.apache.flink.table.planner.factories.TestValuesTableFactory;
import org.apache.flink.types.Row;
+import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -35,6 +37,7 @@ import java.util.Map;
import static org.apache.flink.table.api.DataTypes.BIGINT;
import static org.apache.flink.table.api.DataTypes.DOUBLE;
import static org.apache.flink.table.api.DataTypes.STRING;
+import static org.apache.flink.table.api.DataTypes.TIMESTAMP;
import static org.apache.flink.table.planner.factories.TestValuesTableFactory.changelogRow;
import static org.apache.flink.table.planner.factories.TestValuesTableFactory.registerData;
import static org.apache.flink.table.store.connector.ShowCreateUtil.buildShowCreateTable;
@@ -44,15 +47,29 @@ import static org.apache.flink.table.store.connector.TableStoreTestBase.createRe
public class ReadWriteTableTestUtil {
public static String prepareHelperSourceWithInsertOnlyRecords(
- String sourceTable, List<String> partitionKeys, List<String> primaryKeys) {
+ String sourceTable,
+ List<String> partitionKeys,
+ List<String> primaryKeys,
+ boolean assignWatermark) {
return prepareHelperSourceRecords(
- RuntimeExecutionMode.BATCH, sourceTable, partitionKeys, primaryKeys);
+ RuntimeExecutionMode.BATCH,
+ sourceTable,
+ partitionKeys,
+ primaryKeys,
+ assignWatermark);
}
public static String prepareHelperSourceWithChangelogRecords(
- String sourceTable, List<String> partitionKeys, List<String> primaryKeys) {
+ String sourceTable,
+ List<String> partitionKeys,
+ List<String> primaryKeys,
+ boolean assignWatermark) {
return prepareHelperSourceRecords(
- RuntimeExecutionMode.STREAMING, sourceTable, partitionKeys, primaryKeys);
+ RuntimeExecutionMode.STREAMING,
+ sourceTable,
+ partitionKeys,
+ primaryKeys,
+ assignWatermark);
}
/**
@@ -76,20 +93,23 @@ public class ReadWriteTableTestUtil {
* @param sourceTable source table name
* @param partitionKeys is used to calculate {@code partition-list}
* @param primaryKeys
+ * @param assignWatermark
* @return helper source ddl
*/
private static String prepareHelperSourceRecords(
RuntimeExecutionMode executionMode,
String sourceTable,
List<String> partitionKeys,
- List<String> primaryKeys) {
+ List<String> primaryKeys,
+ boolean assignWatermark) {
Map<String, String> tableOptions = new HashMap<>();
boolean bounded = executionMode == RuntimeExecutionMode.BATCH;
String changelogMode = bounded ? "I" : primaryKeys.size() > 0 ? "I,UA,D" : "I,UA,UB,D";
List<String> exclusivePk = new ArrayList<>(primaryKeys);
exclusivePk.removeAll(partitionKeys);
Tuple2<List<Row>, String> tuple =
- prepareHelperSource(bounded, exclusivePk.size(), partitionKeys.size());
+ prepareHelperSource(
+ bounded, exclusivePk.size(), partitionKeys.size(), assignWatermark);
String dataId = registerData(tuple.f0);
String partitionList = tuple.f1;
tableOptions.put("connector", TestValuesTableFactory.IDENTIFIER);
@@ -101,10 +121,14 @@ public class ReadWriteTableTestUtil {
tableOptions.put("partition-list", partitionList);
}
ResolvedCatalogTable table;
- if (exclusivePk.size() > 1) {
- table = prepareExchangeRateSource(partitionKeys, primaryKeys, tableOptions);
+ if (assignWatermark) {
+ table = prepareRateSourceWithTimestamp(tableOptions);
} else {
- table = prepareRateSource(partitionKeys, primaryKeys, tableOptions);
+ if (exclusivePk.size() > 1) {
+ table = prepareExchangeRateSource(partitionKeys, primaryKeys, tableOptions);
+ } else {
+ table = prepareRateSource(partitionKeys, primaryKeys, tableOptions);
+ }
}
return buildShowCreateTable(
table,
@@ -113,9 +137,12 @@ public class ReadWriteTableTestUtil {
}
private static Tuple2<List<Row>, String> prepareHelperSource(
- boolean bounded, int pkSize, int partitionSize) {
+ boolean bounded, int pkSize, int partitionSize, boolean assignWatermark) {
List<Row> input;
String partitionList = null;
+ if (assignWatermark) {
+ return Tuple2.of(ratesWithTimestamp(), null);
+ }
if (pkSize == 2) {
if (bounded) {
if (partitionSize == 0) {
@@ -220,6 +247,16 @@ public class ReadWriteTableTestUtil {
return createResolvedTable(tableOptions, resolvedColumns, partitionKeys, primaryKeys);
}
+ private static ResolvedCatalogTable prepareRateSourceWithTimestamp(
+ Map<String, String> tableOptions) {
+ List<Column> resolvedColumns = new ArrayList<>();
+ resolvedColumns.add(Column.physical("currency", STRING()));
+ resolvedColumns.add(Column.physical("rate", BIGINT()));
+ resolvedColumns.add(Column.physical("ts", TIMESTAMP(3)));
+ return createResolvedTable(
+ tableOptions, resolvedColumns, Collections.emptyList(), Collections.emptyList());
+ }
+
private static ResolvedCatalogTable prepareExchangeRateSource(
List<String> partitionKeys,
List<String> primaryKeys,
@@ -247,6 +284,15 @@ public class ReadWriteTableTestUtil {
changelogRow("+I", "Euro", 119L));
}
+ public static List<Row> ratesWithTimestamp() {
+ // currency, rate, timestamp
+ return Arrays.asList(
+ changelogRow(
+ "+I", "US Dollar", 102L, LocalDateTime.parse("1990-04-07T10:00:11.120")),
+ changelogRow("+I", "Euro", 119L, LocalDateTime.parse("2020-04-07T10:10:11.120")),
+ changelogRow("+I", "Yen", 1L, LocalDateTime.parse("2022-04-07T09:54:11.120")));
+ }
+
public static List<Row> exchangeRates() {
// from_currency, to_currency, rate_by_to_currency
return Arrays.asList(
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
index a1fa137..1ddb145 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/ShowCreateUtil.java
@@ -49,10 +49,19 @@ public class ShowCreateUtil {
private ShowCreateUtil() {}
public static String createTableLikeDDL(
- String sourceTableName, String managedTableName, Map<String, String> tableOptions) {
+ String sourceTableName,
+ String managedTableName,
+ Map<String, String> tableOptions,
+ @Nullable ReadWriteTableTestBase.WatermarkSpec watermarkSpec) {
StringBuilder ddl =
new StringBuilder("CREATE TABLE IF NOT EXISTS ")
.append(String.format("`%s`", managedTableName));
+ if (watermarkSpec != null) {
+ ddl.append(
+ String.format(
+ "(\n WATERMARK FOR %s AS %s)\n",
+ watermarkSpec.columnName, watermarkSpec.expressionAsString));
+ }
ddl.append(optionsToSql(tableOptions))
.append(String.format(" LIKE `%s` (EXCLUDING OPTIONS)\n", sourceTableName));
return ddl.toString();
[flink-table-store] 05/15: [hotfix] Add random group id to kafka consumer
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit e5d9202d2995398c5c8fd717be36f6b855e2de84
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Fri Apr 8 11:36:05 2022 +0800
[hotfix] Add random group id to kafka consumer
This closes #77
---
.../java/org/apache/flink/table/store/kafka/KafkaLogSourceProvider.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/flink-table-store-kafka/src/main/java/org/apache/flink/table/store/kafka/KafkaLogSourceProvider.java b/flink-table-store-kafka/src/main/java/org/apache/flink/table/store/kafka/KafkaLogSourceProvider.java
index 60308a0..861f77b 100644
--- a/flink-table-store-kafka/src/main/java/org/apache/flink/table/store/kafka/KafkaLogSourceProvider.java
+++ b/flink-table-store-kafka/src/main/java/org/apache/flink/table/store/kafka/KafkaLogSourceProvider.java
@@ -36,6 +36,7 @@ import javax.annotation.Nullable;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
+import java.util.UUID;
import static org.apache.kafka.clients.consumer.ConsumerConfig.ISOLATION_LEVEL_CONFIG;
@@ -108,6 +109,7 @@ public class KafkaLogSourceProvider implements LogSourceProvider {
.setStartingOffsets(toOffsetsInitializer(bucketOffsets))
.setProperties(properties)
.setDeserializer(createDeserializationSchema())
+ .setGroupId(UUID.randomUUID().toString())
.build();
}
[flink-table-store] 08/15: [hotfix] Shade kafka dependencies
Posted by lz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.1
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
commit 7cbef7fe2c3be62e5b85dceedf8b4a230723249a
Author: tsreaper <ts...@gmail.com>
AuthorDate: Fri Apr 8 17:03:11 2022 +0800
[hotfix] Shade kafka dependencies
This closes #81
---
flink-table-store-dist/pom.xml | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/flink-table-store-dist/pom.xml b/flink-table-store-dist/pom.xml
index f745d85..23953b1 100644
--- a/flink-table-store-dist/pom.xml
+++ b/flink-table-store-dist/pom.xml
@@ -208,6 +208,14 @@ under the License.
<pattern>com.google.protobuf</pattern>
<shadedPattern>org.apache.flink.table.store.shaded.com.google.protobuf</shadedPattern>
</relocation>
+ <!--
+ flink-sql-connector-kafka contains shaded kafka classes.
+ As we've shaded kafka connector we also need to shade kafka dependencies.
+ -->
+ <relocation>
+ <pattern>org.apache.kafka</pattern>
+ <shadedPattern>org.apache.flink.table.store.shaded.org.apache.flink.kafka.shaded.org.apache.kafka</shadedPattern>
+ </relocation>
</relocations>
</configuration>
</execution>