You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2023/02/27 09:42:28 UTC

[flink-table-store] branch release-0.3 updated: [FLINK-31201] Scan to generate splits should care about file order

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch release-0.3
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git


The following commit(s) were added to refs/heads/release-0.3 by this push:
     new c07482c3 [FLINK-31201] Scan to generate splits should care about file order
c07482c3 is described below

commit c07482c3e64c576ae53322b4511733e2328cd818
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Feb 27 17:42:21 2023 +0800

    [FLINK-31201] Scan to generate splits should care about file order
    
    This closes #554
---
 .../shortcodes/generated/core_configuration.html   |  6 ++
 .../store/connector/AppendOnlyTableITCase.java     | 16 ++++++
 .../source/FileStoreSourceSplitGeneratorTest.java  | 22 ++++----
 .../org/apache/flink/table/store/CoreOptions.java  | 20 +++++++
 .../table/store/file/operation/FileStoreScan.java  |  6 +-
 .../store/table/source/AbstractDataTableScan.java  | 26 ++++++++-
 .../store/table/AppendOnlyFileStoreTableTest.java  | 64 +++++++++++++++++-----
 7 files changed, 133 insertions(+), 27 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index b123fa65..66c4607d 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -218,6 +218,12 @@
             <td><p>Enum</p></td>
             <td>Specify the scanning behavior of the source.<br /><br />Possible values:<ul><li>"default": Determines actual startup mode according to other table properties. If "scan.timestamp-millis" is set the actual startup mode will be "from-timestamp", and if "scan.snapshot-id" is set the actual startup mode will be "from-snapshot". Otherwise the actual startup mode will be "latest-full".</li><li>"latest-full": For streaming sources, produces the latest snapshot on the table upon f [...]
         </tr>
+        <tr>
+            <td><h5>scan.plan-sort-partition</h5></td>
+            <td style="word-wrap: break-word;">false</td>
+            <td>Boolean</td>
+            <td>Whether to sort plan files by partition fields, this allows you to read according to the partition order, even if your partition writes are out of order.<br />It is recommended that you use this for streaming read of the 'append-only' table. By default, streaming read will read the full snapshot first. In order to avoid the disorder reading for partitions, you can open this option.</td>
+        </tr>
         <tr>
             <td><h5>scan.snapshot-id</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/AppendOnlyTableITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/AppendOnlyTableITCase.java
index cfee8bd2..f5690522 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/AppendOnlyTableITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/AppendOnlyTableITCase.java
@@ -19,6 +19,7 @@
 package org.apache.flink.table.store.connector;
 
 import org.apache.flink.table.api.TableException;
+import org.apache.flink.table.api.config.ExecutionConfigOptions;
 import org.apache.flink.table.planner.factories.TestValuesTableFactory;
 import org.apache.flink.table.store.file.Snapshot;
 import org.apache.flink.types.Row;
@@ -71,6 +72,20 @@ public class AppendOnlyTableITCase extends FileStoreTableITCase {
         assertThat(rows).containsExactlyInAnyOrder(Row.of("AAA"), Row.of("BBB"));
     }
 
+    @Test
+    public void testReadPartitionOrder() {
+        bEnv.getConfig().set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 1);
+        batchSql("INSERT INTO part_table VALUES (1, 'AAA', 'part-1')");
+        batchSql("INSERT INTO part_table VALUES (2, 'BBB', 'part-2')");
+        batchSql("INSERT INTO part_table VALUES (3, 'CCC', 'part-3')");
+
+        assertThat(batchSql("SELECT * FROM part_table"))
+                .containsExactly(
+                        Row.of(1, "AAA", "part-1"),
+                        Row.of(2, "BBB", "part-2"),
+                        Row.of(3, "CCC", "part-3"));
+    }
+
     @Test
     public void testSkipDedup() {
         batchSql("INSERT INTO append_table VALUES (1, 'AAA'), (1, 'AAA'), (2, 'BBB'), (3, 'BBB')");
@@ -206,6 +221,7 @@ public class AppendOnlyTableITCase extends FileStoreTableITCase {
     protected List<String> ddl() {
         return Arrays.asList(
                 "CREATE TABLE IF NOT EXISTS append_table (id INT, data STRING) WITH ('write-mode'='append-only')",
+                "CREATE TABLE IF NOT EXISTS part_table (id INT, data STRING, dt STRING) PARTITIONED BY (dt) WITH ('write-mode'='append-only')",
                 "CREATE TABLE IF NOT EXISTS complex_table (id INT, data MAP<INT, INT>) WITH ('write-mode'='append-only')");
     }
 
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
index 8d7b1937..7c4b5969 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/source/FileStoreSourceSplitGeneratorTest.java
@@ -95,18 +95,20 @@ public class FileStoreSourceSplitGeneratorTest {
                                                 .getInt(0))
                         .thenComparing(
                                 o -> ((DataSplit) ((FileStoreSourceSplit) o).split()).bucket()));
-        assertSplit(splits.get(0), "0000000007", 1, 0, Arrays.asList("f0", "f1"));
-        assertSplit(splits.get(1), "0000000008", 1, 1, Collections.singletonList("f2"));
+
+        // splitId should be the input order!
+        assertSplit(splits.get(0), "0000000001", 1, 0, Arrays.asList("f0", "f1"));
+        assertSplit(splits.get(1), "0000000002", 1, 1, Collections.singletonList("f2"));
         assertSplit(splits.get(2), "0000000003", 2, 0, Arrays.asList("f3", "f4", "f5"));
         assertSplit(splits.get(3), "0000000004", 2, 1, Collections.singletonList("f6"));
-        assertSplit(splits.get(4), "0000000001", 3, 0, Collections.singletonList("f7"));
-        assertSplit(splits.get(5), "0000000002", 3, 1, Collections.singletonList("f8"));
-        assertSplit(splits.get(6), "0000000011", 4, 0, Collections.singletonList("f9"));
-        assertSplit(splits.get(7), "0000000012", 4, 1, Collections.singletonList("f10"));
-        assertSplit(splits.get(8), "0000000005", 5, 0, Collections.singletonList("f11"));
-        assertSplit(splits.get(9), "0000000006", 5, 1, Collections.singletonList("f12"));
-        assertSplit(splits.get(10), "0000000009", 6, 0, Collections.singletonList("f13"));
-        assertSplit(splits.get(11), "0000000010", 6, 1, Collections.singletonList("f14"));
+        assertSplit(splits.get(4), "0000000005", 3, 0, Collections.singletonList("f7"));
+        assertSplit(splits.get(5), "0000000006", 3, 1, Collections.singletonList("f8"));
+        assertSplit(splits.get(6), "0000000007", 4, 0, Collections.singletonList("f9"));
+        assertSplit(splits.get(7), "0000000008", 4, 1, Collections.singletonList("f10"));
+        assertSplit(splits.get(8), "0000000009", 5, 0, Collections.singletonList("f11"));
+        assertSplit(splits.get(9), "0000000010", 5, 1, Collections.singletonList("f12"));
+        assertSplit(splits.get(10), "0000000011", 6, 0, Collections.singletonList("f13"));
+        assertSplit(splits.get(11), "0000000012", 6, 1, Collections.singletonList("f14"));
     }
 
     private void assertSplit(
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
index c5c3b1e6..f19eb12c 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CoreOptions.java
@@ -456,6 +456,22 @@ public class CoreOptions implements Serializable {
                                                             + "$hour:00:00'."))
                                     .build());
 
+    public static final ConfigOption<Boolean> SCAN_PLAN_SORT_PARTITION =
+            ConfigOptions.key("scan.plan-sort-partition")
+                    .booleanType()
+                    .defaultValue(false)
+                    .withDescription(
+                            Description.builder()
+                                    .text(
+                                            "Whether to sort plan files by partition fields, this allows you to read"
+                                                    + " according to the partition order, even if your partition writes are out of order.")
+                                    .linebreak()
+                                    .text(
+                                            "It is recommended that you use this for streaming read of the 'append-only' table."
+                                                    + " By default, streaming read will read the full snapshot first. In order to"
+                                                    + " avoid the disorder reading for partitions, you can open this option.")
+                                    .build());
+
     private final Configuration options;
 
     public CoreOptions(Map<String, String> options) {
@@ -606,6 +622,10 @@ public class CoreOptions implements Serializable {
         return options.get(CHANGELOG_PRODUCER);
     }
 
+    public boolean scanPlanSortPartition() {
+        return options.get(SCAN_PLAN_SORT_PARTITION);
+    }
+
     public StartupMode startupMode() {
         return startupMode(options);
     }
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
index b9fe5f80..0f375ad6 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/operation/FileStoreScan.java
@@ -28,7 +28,7 @@ import org.apache.flink.table.store.file.predicate.Predicate;
 import javax.annotation.Nullable;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -74,9 +74,9 @@ public interface FileStoreScan {
         /** Return a map group by partition and bucket. */
         default Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> groupByPartFiles(
                 List<ManifestEntry> files) {
-            Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> groupBy = new HashMap<>();
+            Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> groupBy = new LinkedHashMap<>();
             for (ManifestEntry entry : files) {
-                groupBy.computeIfAbsent(entry.partition(), k -> new HashMap<>())
+                groupBy.computeIfAbsent(entry.partition(), k -> new LinkedHashMap<>())
                         .computeIfAbsent(entry.bucket(), k -> new ArrayList<>())
                         .add(entry.file());
             }
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
index 73f9fcc2..8de885ee 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/source/AbstractDataTableScan.java
@@ -21,6 +21,8 @@ package org.apache.flink.table.store.table.source;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.table.data.binary.BinaryRowData;
 import org.apache.flink.table.store.CoreOptions;
+import org.apache.flink.table.store.codegen.CodeGenUtils;
+import org.apache.flink.table.store.codegen.RecordComparator;
 import org.apache.flink.table.store.file.Snapshot;
 import org.apache.flink.table.store.file.io.DataFileMeta;
 import org.apache.flink.table.store.file.manifest.FileKind;
@@ -33,6 +35,7 @@ import org.apache.flink.table.store.file.utils.FileStorePathFactory;
 import org.apache.flink.table.store.file.utils.SnapshotManager;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -49,6 +52,8 @@ public abstract class AbstractDataTableScan implements DataTableScan {
 
     private ScanKind scanKind = ScanKind.ALL;
 
+    private RecordComparator lazyPartitionComparator;
+
     protected AbstractDataTableScan(
             FileStoreScan scan,
             TableSchema tableSchema,
@@ -113,17 +118,36 @@ public abstract class AbstractDataTableScan implements DataTableScan {
         return this;
     }
 
+    private RecordComparator partitionComparator() {
+        if (lazyPartitionComparator == null) {
+            lazyPartitionComparator =
+                    CodeGenUtils.newRecordComparator(
+                            tableSchema.logicalPartitionType().getChildren(),
+                            "PartitionComparator");
+        }
+        return lazyPartitionComparator;
+    }
+
     @Override
     public DataFilePlan plan() {
         FileStoreScan.Plan plan = scan.plan();
         Long snapshotId = plan.snapshotId();
 
+        Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> files =
+                plan.groupByPartFiles(plan.files(FileKind.ADD));
+        if (options.scanPlanSortPartition()) {
+            Map<BinaryRowData, Map<Integer, List<DataFileMeta>>> newFiles = new LinkedHashMap<>();
+            files.entrySet().stream()
+                    .sorted((o1, o2) -> partitionComparator().compare(o1.getKey(), o2.getKey()))
+                    .forEach(entry -> newFiles.put(entry.getKey(), entry.getValue()));
+            files = newFiles;
+        }
         List<DataSplit> splits =
                 generateSplits(
                         snapshotId == null ? Snapshot.FIRST_SNAPSHOT_ID - 1 : snapshotId,
                         scanKind != ScanKind.ALL,
                         splitGenerator(pathFactory),
-                        plan.groupByPartFiles(plan.files(FileKind.ADD)));
+                        files);
         return new DataFilePlan(snapshotId, splits);
     }
 
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTableTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTableTest.java
index e1cca63f..cf834355 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTableTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/AppendOnlyFileStoreTableTest.java
@@ -33,6 +33,7 @@ import org.apache.flink.table.store.file.schema.UpdateSchema;
 import org.apache.flink.table.store.table.sink.TableCommit;
 import org.apache.flink.table.store.table.sink.TableWrite;
 import org.apache.flink.table.store.table.source.AbstractDataTableScan;
+import org.apache.flink.table.store.table.source.DataSplit;
 import org.apache.flink.table.store.table.source.Split;
 import org.apache.flink.table.store.table.source.TableRead;
 
@@ -112,21 +113,58 @@ public class AppendOnlyFileStoreTableTest extends FileStoreTableTestBase {
     }
 
     @Test
-    public void testStreamingReadWrite() throws Exception {
-        writeData();
+    public void testBatchPlanOrder() throws Exception {
         FileStoreTable table = createFileStoreTable();
 
-        List<Split> splits = table.newScan().withKind(ScanKind.DELTA).plan().splits();
-        TableRead read = table.newRead();
-        assertThat(getResult(read, splits, binaryRow(1), 0, STREAMING_ROW_TO_STRING))
-                .isEqualTo(
-                        Arrays.asList(
-                                "+1|11|101|binary|varbinary|mapKey:mapVal|multiset",
-                                "+1|12|102|binary|varbinary|mapKey:mapVal|multiset"));
-        assertThat(getResult(read, splits, binaryRow(2), 0, STREAMING_ROW_TO_STRING))
-                .isEqualTo(
-                        Collections.singletonList(
-                                "+2|21|201|binary|varbinary|mapKey:mapVal|multiset"));
+        TableWrite write = table.newWrite(commitUser);
+        TableCommit commit = table.newCommit(commitUser);
+
+        write.write(rowData(1, 10, 100L));
+        commit.commit(0, write.prepareCommit(true, 0));
+
+        write.write(rowData(2, 22, 202L));
+        commit.commit(1, write.prepareCommit(true, 1));
+
+        write.write(rowData(3, 33, 303L));
+        commit.commit(2, write.prepareCommit(true, 2));
+        write.close();
+
+        List<Split> splits = table.newScan().plan().splits();
+        int[] partitions =
+                splits.stream()
+                        .map(split -> ((DataSplit) split).partition().getInt(0))
+                        .mapToInt(Integer::intValue)
+                        .toArray();
+        assertThat(partitions).containsExactly(1, 2, 3);
+    }
+
+    @Test
+    public void testBatchPlanOrderByPartition() throws Exception {
+        FileStoreTable table =
+                createFileStoreTable(
+                        options -> options.set(CoreOptions.SCAN_PLAN_SORT_PARTITION, true));
+
+        TableWrite write = table.newWrite(commitUser);
+        TableCommit commit = table.newCommit(commitUser);
+
+        write.write(rowData(3, 33, 303L));
+        commit.commit(0, write.prepareCommit(true, 0));
+
+        write.write(rowData(1, 10, 100L));
+        commit.commit(1, write.prepareCommit(true, 1));
+
+        write.write(rowData(2, 22, 202L));
+        commit.commit(2, write.prepareCommit(true, 2));
+
+        write.close();
+
+        List<Split> splits = table.newScan().plan().splits();
+        int[] partitions =
+                splits.stream()
+                        .map(split -> ((DataSplit) split).partition().getInt(0))
+                        .mapToInt(Integer::intValue)
+                        .toArray();
+        assertThat(partitions).containsExactly(1, 2, 3);
     }
 
     @Test