You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2020/12/09 04:30:44 UTC
[flink] 01/02: [FLINK-20419][table-planner-blink] Avoid dynamic
partition grouping if the input defines collation
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 5a55f0173a26dfc7af250d7aed55f81a00660978
Author: Rui Li <li...@apache.org>
AuthorDate: Thu Dec 3 15:41:18 2020 +0800
[FLINK-20419][table-planner-blink] Avoid dynamic partition grouping if the input defines collation
This closes #14294
---
.../plan/rules/physical/batch/BatchExecSinkRule.scala | 15 +++++++++++----
.../planner/factories/TestValuesTableFactory.java | 13 ++++++++++++-
.../table/planner/plan/batch/sql/TableSinkTest.xml | 19 +++++++++++++++++++
.../table/planner/plan/batch/sql/TableSinkTest.scala | 18 ++++++++++++++++++
4 files changed, 60 insertions(+), 5 deletions(-)
diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchExecSinkRule.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchExecSinkRule.scala
index 2d42930..53ed66ee 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchExecSinkRule.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchExecSinkRule.scala
@@ -30,7 +30,7 @@ import org.apache.flink.table.types.logical.RowType
import org.apache.calcite.plan.RelOptRule
import org.apache.calcite.rel.convert.ConverterRule
-import org.apache.calcite.rel.{RelCollations, RelNode}
+import org.apache.calcite.rel.{RelCollationTraitDef, RelCollations, RelNode}
import scala.collection.JavaConversions._
@@ -72,9 +72,16 @@ class BatchExecSinkRule extends ConverterRule(
}
if (partitionSink.requiresPartitionGrouping(true)) {
- // default to asc.
- val fieldCollations = dynamicPartIndices.map(FlinkRelOptUtil.ofRelFieldCollation)
- requiredTraitSet = requiredTraitSet.plus(RelCollations.of(fieldCollations: _*))
+ // we shouldn't do partition grouping if the input already defines collation
+ val relCollation = requiredTraitSet.getTrait(RelCollationTraitDef.INSTANCE)
+ if (relCollation == null || relCollation.getFieldCollations.isEmpty) {
+ // default to asc.
+ val fieldCollations = dynamicPartIndices.map(FlinkRelOptUtil.ofRelFieldCollation)
+ requiredTraitSet = requiredTraitSet.plus(RelCollations.of(fieldCollations: _*))
+ } else {
+ // tell sink not to expect grouping
+ partitionSink.requiresPartitionGrouping(false)
+ }
}
}
case _ => throw new TableException(
diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
index a12491e..292c8d0 100644
--- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
+++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
@@ -42,6 +42,7 @@ import org.apache.flink.table.connector.sink.DataStreamSinkProvider;
import org.apache.flink.table.connector.sink.DynamicTableSink;
import org.apache.flink.table.connector.sink.OutputFormatProvider;
import org.apache.flink.table.connector.sink.SinkFunctionProvider;
+import org.apache.flink.table.connector.sink.abilities.SupportsPartitioning;
import org.apache.flink.table.connector.sink.abilities.SupportsWritingMetadata;
import org.apache.flink.table.connector.source.AsyncTableFunctionProvider;
import org.apache.flink.table.connector.source.DataStreamScanProvider;
@@ -1094,7 +1095,8 @@ public final class TestValuesTableFactory implements DynamicTableSourceFactory,
*/
private static class TestValuesTableSink implements
DynamicTableSink,
- SupportsWritingMetadata {
+ SupportsWritingMetadata,
+ SupportsPartitioning {
private DataType consumedDataType;
private int[] primaryKeyIndices;
@@ -1254,6 +1256,15 @@ public final class TestValuesTableFactory implements DynamicTableSourceFactory,
public void applyWritableMetadata(List<String> metadataKeys, DataType consumedDataType) {
this.consumedDataType = consumedDataType;
}
+
+ @Override
+ public void applyStaticPartition(Map<String, String> partition) {
+ }
+
+ @Override
+ public boolean requiresPartitionGrouping(boolean supportsGrouping) {
+ return supportsGrouping;
+ }
}
/**
diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.xml
index c1d923e..986207f 100644
--- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.xml
+++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.xml
@@ -79,4 +79,23 @@ Sink(table=[default_catalog.default_database.sink2], fields=[total_min])
]]>
</Resource>
</TestCase>
+ <TestCase name="testDynamicPartWithOrderBy">
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalSink(table=[default_catalog.default_database.sink], fields=[a, b])
++- LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])
+ +- LogicalProject(a=[$0], b=[$1])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+Sink(table=[default_catalog.default_database.sink], fields=[a, b])
++- Sort(orderBy=[a ASC])
+ +- Exchange(distribution=[single])
+ +- Calc(select=[a, b])
+ +- BoundedStreamScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c])
+]]>
+ </Resource>
+ </TestCase>
</Root>
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala
index fc544da..32a95fb 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala
@@ -78,4 +78,22 @@ class TableSinkTest extends TableTestBase {
util.verifyPlan(stmtSet)
}
+
+ @Test
+ def testDynamicPartWithOrderBy(): Unit = {
+ util.addTable(
+ s"""
+ |CREATE TABLE sink (
+ | `a` INT,
+ | `b` BIGINT
+ |) PARTITIONED BY (
+ | `b`
+ |) WITH (
+ | 'connector' = 'values'
+ |)
+ |""".stripMargin)
+ val stmtSet = util.tableEnv.createStatementSet()
+ stmtSet.addInsertSql("INSERT INTO sink SELECT a,b FROM MyTable ORDER BY a")
+ util.verifyPlan(stmtSet)
+ }
}