You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2023/02/27 09:42:28 UTC
[flink-table-store] branch release-0.3 updated: [FLINK-31201] Scan to generate splits should care about file order
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.3
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
The following commit(s) were added to refs/heads/release-0.3 by this push:
new c07482c3 [FLINK-31201] Scan to generate splits should care about file order
c07482c3 is described below
commit c07482c3e64c576ae53322b4511733e2328cd818
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Feb 27 17:42:21 2023 +0800
[FLINK-31201] Scan to generate splits should care about file order
This closes #554
---
.../shortcodes/generated/core_configuration.html | 6 ++
.../store/connector/AppendOnlyTableITCase.java | 16 ++++++
.../source/FileStoreSourceSplitGeneratorTest.java | 22 ++++----
.../org/apache/flink/table/store/CoreOptions.java | 20 +++++++
.../table/store/file/operation/FileStoreScan.java | 6 +-
.../store/table/source/AbstractDataTableScan.java | 26 ++++++++-
.../store/table/AppendOnlyFileStoreTableTest.java | 64 +++++++++++++++++-----
7 files changed, 133 insertions(+), 27 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index b123fa65..66c4607d 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -218,6 +218,12 @@
<td><p>Enum</p></td>
<td>Specify the scanning behavior of the source.<br /><br />Possible values:<ul><li>"default": Determines actual startup mode according to other table properties. If "scan.timestamp-millis" is set the actual startup mode will be "from-timestamp", and if "scan.snapshot-id" is set the actual startup mode will be "from-snapshot". Otherwise the actual startup mode will be "latest-full".</li><li>"latest-full": For streaming sources, produces the latest snapshot on the table upon f [...]
</tr>
+ <tr>
+ <td><h5>scan.plan-sort-partition</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Whether to sort plan files by partition fields, this allows you to read according to the partition order, even if your partition writes are out of order.<br />It is recommended that you use this for streaming read of the 'append-only' table. By default, streaming read will read the full snapshot first. In order to avoid the disorder reading for partitions, you can open this option.</td>
+ </tr>
<tr>
<td><h5>scan.snapshot-id</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/AppendOnlyTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/AppendOnlyTableITCase.java
index cfee8bd2..f5690522 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/AppendOnlyTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/AppendOnlyTableITCase.java
@@ -19,6 +19,7 @@
package org.apache.flink.table.store.connector;
import org.apache.flink.table.api.TableException;
+import org.apache.flink.table.api.config.ExecutionConfigOptions;
import org.apache.flink.table.planner.factories.TestValuesTableFactory;
import org.apache.flink.table.store.file.Snapshot;
import org.apache.flink.types.Row;
@@ -71,6 +72,20 @@ public class AppendOnlyTableITCase extends FileStoreTableITCase {
assertThat(rows).containsExactlyInAnyOrder(Row.of("AAA"), Row.of("BBB"));
}
+ @Test
+ public void testReadPartitionOrder() {
+ bEnv.getConfig().set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 1);
+ batchSql("INSERT INTO part_table VALUES (1, 'AAA', 'part-1')");
+ batchSql("INSERT INTO part_table VALUES (2, 'BBB', 'part-2')");
+ batchSql("INSERT INTO part_table VALUES (3, 'CCC', 'part-3')");
+
+ assertThat(batchSql("SELECT * FROM part_table"))
+ .containsExactly(
+ Row.of(1, "AAA", "part-1"),
+ Row.of(2, "BBB", "part-2"),
+ Row.of(3, "CCC", "part-3"));
+ }
+
@Test
public void testSkipDedup() {
batchSql("INSERT INTO append_table VALUES (1, 'AAA'), (1, 'AAA'), (2, 'BBB'), (3, 'BBB')");
@@ -206,6 +221,7 @@ public class AppendOnlyTableITCase extends FileStoreTableITCase {
protected List<String> ddl() {
return Arrays.asList(
"CREATE TABLE IF NOT EXISTS append_table (id INT, data STRING) WITH ('write-mode'='append-only')",
+ "CREATE TABLE IF NOT EXISTS part_table (id INT, data STRING, dt STRING) PARTITIONED BY (dt) WITH ('write-mode'='append-only')",
"CREATE TABLE IF NOT EXISTS complex_table (id INT, data MAP<INT, INT>) WITH ('write-mode'='append-only')");
}
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
index 8d7b1937..7c4b5969 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
@@ -95,18 +95,20 @@ public class FileStoreSourceSplitGeneratorTest {
.getInt(0))
.thenComparing(
o -> ((DataSplit) ((FileStoreSourceSplit) o).split()).bucket()));
- assertSplit(splits.get(0), "0000000007", 1, 0, Arrays.asList("f0", "f1"));
- assertSplit(splits.get(1), "0000000008", 1, 1, Collections.singletonList("f2"));
+
+ // splitId should be the input order!
+ assertSplit(splits.get(0), "0000000001", 1, 0, Arrays.asList("f0", "f1"));
+ assertSplit(splits.get(1), "0000000002", 1, 1, Collections.singletonList("f2"));
assertSplit(splits.get(2), "0000000003", 2, 0, Arrays.asList("f3", "f4", "f5"));
assertSplit(splits.get(3), "0000000004", 2, 1, Collections.singletonList("f6"));
- assertSplit(splits.get(4), "0000000001", 3, 0, Collections.singletonList("f7"));
- assertSplit(splits.get(5), "0000000002", 3, 1, Collections.singletonList("f8"));
- assertSplit(splits.get(6), "0000000011", 4, 0, Collections.singletonList("f9"));
- assertSplit(splits.get(7), "0000000012", 4, 1, Collections.singletonList("f10"));
- assertSplit(splits.get(8), "0000000005", 5, 0, Collections.singletonList("f11"));
- assertSplit(splits.get(9), "0000000006", 5, 1, Collections.singletonList("f12"));
- assertSplit(splits.get(10), "0000000009", 6, 0, Collections.singletonList("f13"));
- assertSplit(splits.get(11), "0000000010", 6, 1, Collections.singletonList("f14"));
+ assertSplit(splits.get(4), "0000000005", 3, 0, Collections.singletonList("f7"));
+ assertSplit(splits.get(5), "0000000006", 3, 1, Collections.singletonList("f8"));
+ assertSplit(splits.get(6), "0000000007", 4, 0, Collections.singletonList("f9"));
+ assertSplit(splits.get(7), "0000000008", 4, 1, Collections.singletonList("f10"));
+ assertSplit(splits.get(8), "0000000009", 5, 0, Collections.singletonList("f11"));
+ assertSplit(splits.get(9), "0000000010", 5, 1, Collections.singletonList("f12"));
+ assertSplit(splits.get(10), "0000000011", 6, 0, Collections.singletonList("f13"));
+ assertSplit(splits.get(11), "0000000012", 6, 1, Collections.singletonList("f14"));
}
private void assertSplit(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
index c5c3b1e6..f19eb12c 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
@@ -456,6 +456,22 @@ public class CoreOptions implements Serializable {
+ "$hour:00:00'."))
.build());
+ public static final ConfigOption<Boolean> SCAN_PLAN_SORT_PARTITION =
+ ConfigOptions.key("scan.plan-sort-partition")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ Description.builder()
+ .text(
+ "Whether to sort plan files by partition fields, this allows you to read"
+ + " according to the partition order, even if your partition writes are out of order.")
+ .linebreak()
+ .text(
+ "It is recommended that you use this for streaming read of the 'append-only' table."
+ + " By default, streaming read will read the full snapshot first. In order to"
+ + " avoid the disorder reading for partitions, you can open this option.")
+ .build());
+
private final Configuration options;
public CoreOptions(Map<String, String> options) {
@@ -606,6 +622,10 @@ public class CoreOptions implements Serializable {
return options.get(CHANGELOG_PRODUCER);
}
+ public boolean scanPlanSortPartition() {
+ return options.get(SCAN_PLAN_SORT_PARTITION);
+ }
+
public StartupMode startupMode() {
return startupMode(options);
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
index b9fe5f80..0f375ad6 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
@@ -28,7 +28,7 @@ import org.apache.flink.table.store.file.predicate.Predicate;
import javax.annotation.Nullable;
import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -74,9 +74,9 @@ public interface FileStoreScan {
/** Return a map group by partition and bucket. */
default Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> groupByPartFiles(
List<ManifestEntry> files) {
- Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> groupBy = new HashMap<>();
+ Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> groupBy = new LinkedHashMap<>();
for (ManifestEntry entry : files) {
- groupBy.computeIfAbsent(entry.partition(), k -> new HashMap<>())
+ groupBy.computeIfAbsent(entry.partition(), k -> new LinkedHashMap<>())
.computeIfAbsent(entry.bucket(), k -> new ArrayList<>())
.add(entry.file());
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
index 73f9fcc2..8de885ee 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
@@ -21,6 +21,8 @@ package org.apache.flink.table.store.table.source;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.table.data.binary.BinaryRowData;
import org.apache.flink.table.store.CoreOptions;
+import org.apache.flink.table.store.codegen.CodeGenUtils;
+import org.apache.flink.table.store.codegen.RecordComparator;
import org.apache.flink.table.store.file.Snapshot;
import org.apache.flink.table.store.file.io.DataFileMeta;
import org.apache.flink.table.store.file.manifest.FileKind;
@@ -33,6 +35,7 @@ import org.apache.flink.table.store.file.utils.FileStorePathFactory;
import org.apache.flink.table.store.file.utils.SnapshotManager;
import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -49,6 +52,8 @@ public abstract class AbstractDataTableScan implements DataTableScan {
private ScanKind scanKind = ScanKind.ALL;
+ private RecordComparator lazyPartitionComparator;
+
protected AbstractDataTableScan(
FileStoreScan scan,
TableSchema tableSchema,
@@ -113,17 +118,36 @@ public abstract class AbstractDataTableScan implements DataTableScan {
return this;
}
+ private RecordComparator partitionComparator() {
+ if (lazyPartitionComparator == null) {
+ lazyPartitionComparator =
+ CodeGenUtils.newRecordComparator(
+ tableSchema.logicalPartitionType().getChildren(),
+ "PartitionComparator");
+ }
+ return lazyPartitionComparator;
+ }
+
@Override
public DataFilePlan plan() {
FileStoreScan.Plan plan = scan.plan();
Long snapshotId = plan.snapshotId();
+ Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> files =
+ plan.groupByPartFiles(plan.files(FileKind.ADD));
+ if (options.scanPlanSortPartition()) {
+ Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> newFiles = new LinkedHashMap<>();
+ files.entrySet().stream()
+ .sorted((o1, o2) -> partitionComparator().compare(o1.getKey(), o2.getKey()))
+ .forEach(entry -> newFiles.put(entry.getKey(), entry.getValue()));
+ files = newFiles;
+ }
List<DataSplit> splits =
generateSplits(
snapshotId == null ? Snapshot.FIRST_SNAPSHOT_ID - 1 : snapshotId,
scanKind != ScanKind.ALL,
splitGenerator(pathFactory),
- plan.groupByPartFiles(plan.files(FileKind.ADD)));
+ files);
return new DataFilePlan(snapshotId, splits);
}
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTableTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTableTest.java
index e1cca63f..cf834355 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTableTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTableTest.java
@@ -33,6 +33,7 @@ import org.apache.flink.table.store.file.schema.UpdateSchema;
import org.apache.flink.table.store.table.sink.TableCommit;
import org.apache.flink.table.store.table.sink.TableWrite;
import org.apache.flink.table.store.table.source.AbstractDataTableScan;
+import org.apache.flink.table.store.table.source.DataSplit;
import org.apache.flink.table.store.table.source.Split;
import org.apache.flink.table.store.table.source.TableRead;
@@ -112,21 +113,58 @@ public class AppendOnlyFileStoreTableTest extends FileStoreTableTestBase {
}
@Test
- public void testStreamingReadWrite() throws Exception {
- writeData();
+ public void testBatchPlanOrder() throws Exception {
FileStoreTable table = createFileStoreTable();
- List<Split> splits = table.newScan().withKind(ScanKind.DELTA).plan().splits();
- TableRead read = table.newRead();
- assertThat(getResult(read, splits, binaryRow(1), 0, STREAMING_ROW_TO_STRING))
- .isEqualTo(
- Arrays.asList(
- "+1|11|101|binary|varbinary|mapKey:mapVal|multiset",
- "+1|12|102|binary|varbinary|mapKey:mapVal|multiset"));
- assertThat(getResult(read, splits, binaryRow(2), 0, STREAMING_ROW_TO_STRING))
- .isEqualTo(
- Collections.singletonList(
- "+2|21|201|binary|varbinary|mapKey:mapVal|multiset"));
+ TableWrite write = table.newWrite(commitUser);
+ TableCommit commit = table.newCommit(commitUser);
+
+ write.write(rowData(1, 10, 100L));
+ commit.commit(0, write.prepareCommit(true, 0));
+
+ write.write(rowData(2, 22, 202L));
+ commit.commit(1, write.prepareCommit(true, 1));
+
+ write.write(rowData(3, 33, 303L));
+ commit.commit(2, write.prepareCommit(true, 2));
+ write.close();
+
+ List<Split> splits = table.newScan().plan().splits();
+ int[] partitions =
+ splits.stream()
+ .map(split -> ((DataSplit) split).partition().getInt(0))
+ .mapToInt(Integer::intValue)
+ .toArray();
+ assertThat(partitions).containsExactly(1, 2, 3);
+ }
+
+ @Test
+ public void testBatchPlanOrderByPartition() throws Exception {
+ FileStoreTable table =
+ createFileStoreTable(
+ options -> options.set(CoreOptions.SCAN_PLAN_SORT_PARTITION, true));
+
+ TableWrite write = table.newWrite(commitUser);
+ TableCommit commit = table.newCommit(commitUser);
+
+ write.write(rowData(3, 33, 303L));
+ commit.commit(0, write.prepareCommit(true, 0));
+
+ write.write(rowData(1, 10, 100L));
+ commit.commit(1, write.prepareCommit(true, 1));
+
+ write.write(rowData(2, 22, 202L));
+ commit.commit(2, write.prepareCommit(true, 2));
+
+ write.close();
+
+ List<Split> splits = table.newScan().plan().splits();
+ int[] partitions =
+ splits.stream()
+ .map(split -> ((DataSplit) split).partition().getInt(0))
+ .mapToInt(Integer::intValue)
+ .toArray();
+ assertThat(partitions).containsExactly(1, 2, 3);
}
@Test