You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2020/12/09 04:30:44 UTC

[flink] 01/02: [FLINK-20419][table-planner-blink] Avoid dynamic partition grouping if the input defines collation

This is an automated email from the ASF dual-hosted git repository.

jark pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 5a55f0173a26dfc7af250d7aed55f81a00660978
Author: Rui Li <li...@apache.org>
AuthorDate: Thu Dec 3 15:41:18 2020 +0800

    [FLINK-20419][table-planner-blink] Avoid dynamic partition grouping if the input defines collation
    
    This closes #14294
---
 .../plan/rules/physical/batch/BatchExecSinkRule.scala | 15 +++++++++++----
 .../planner/factories/TestValuesTableFactory.java     | 13 ++++++++++++-
 .../table/planner/plan/batch/sql/TableSinkTest.xml    | 19 +++++++++++++++++++
 .../table/planner/plan/batch/sql/TableSinkTest.scala  | 18 ++++++++++++++++++
 4 files changed, 60 insertions(+), 5 deletions(-)

diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchExecSinkRule.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchExecSinkRule.scala
index 2d42930..53ed66ee 100644
--- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchExecSinkRule.scala
+++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchExecSinkRule.scala
@@ -30,7 +30,7 @@ import org.apache.flink.table.types.logical.RowType
 
 import org.apache.calcite.plan.RelOptRule
 import org.apache.calcite.rel.convert.ConverterRule
-import org.apache.calcite.rel.{RelCollations, RelNode}
+import org.apache.calcite.rel.{RelCollationTraitDef, RelCollations, RelNode}
 
 import scala.collection.JavaConversions._
 
@@ -72,9 +72,16 @@ class BatchExecSinkRule extends ConverterRule(
             }
 
             if (partitionSink.requiresPartitionGrouping(true)) {
-              // default to asc.
-              val fieldCollations = dynamicPartIndices.map(FlinkRelOptUtil.ofRelFieldCollation)
-              requiredTraitSet = requiredTraitSet.plus(RelCollations.of(fieldCollations: _*))
+              // we shouldn't do partition grouping if the input already defines collation
+              val relCollation = requiredTraitSet.getTrait(RelCollationTraitDef.INSTANCE)
+              if (relCollation == null || relCollation.getFieldCollations.isEmpty) {
+                // default to asc.
+                val fieldCollations = dynamicPartIndices.map(FlinkRelOptUtil.ofRelFieldCollation)
+                requiredTraitSet = requiredTraitSet.plus(RelCollations.of(fieldCollations: _*))
+              } else {
+                // tell sink not to expect grouping
+                partitionSink.requiresPartitionGrouping(false)
+              }
             }
           }
         case _ => throw new TableException(
diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
index a12491e..292c8d0 100644
--- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
+++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/factories/TestValuesTableFactory.java
@@ -42,6 +42,7 @@ import org.apache.flink.table.connector.sink.DataStreamSinkProvider;
 import org.apache.flink.table.connector.sink.DynamicTableSink;
 import org.apache.flink.table.connector.sink.OutputFormatProvider;
 import org.apache.flink.table.connector.sink.SinkFunctionProvider;
+import org.apache.flink.table.connector.sink.abilities.SupportsPartitioning;
 import org.apache.flink.table.connector.sink.abilities.SupportsWritingMetadata;
 import org.apache.flink.table.connector.source.AsyncTableFunctionProvider;
 import org.apache.flink.table.connector.source.DataStreamScanProvider;
@@ -1094,7 +1095,8 @@ public final class TestValuesTableFactory implements DynamicTableSourceFactory,
 	 */
 	private static class TestValuesTableSink implements
 			DynamicTableSink,
-			SupportsWritingMetadata {
+			SupportsWritingMetadata,
+			SupportsPartitioning {
 
 		private DataType consumedDataType;
 		private int[] primaryKeyIndices;
@@ -1254,6 +1256,15 @@ public final class TestValuesTableFactory implements DynamicTableSourceFactory,
 		public void applyWritableMetadata(List<String> metadataKeys, DataType consumedDataType) {
 			this.consumedDataType = consumedDataType;
 		}
+
+		@Override
+		public void applyStaticPartition(Map<String, String> partition) {
+		}
+
+		@Override
+		public boolean requiresPartitionGrouping(boolean supportsGrouping) {
+			return supportsGrouping;
+		}
 	}
 
 	/**
diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.xml
index c1d923e..986207f 100644
--- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.xml
+++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.xml
@@ -79,4 +79,23 @@ Sink(table=[default_catalog.default_database.sink2], fields=[total_min])
 ]]>
     </Resource>
   </TestCase>
+  <TestCase name="testDynamicPartWithOrderBy">
+    <Resource name="planBefore">
+      <![CDATA[
+LogicalSink(table=[default_catalog.default_database.sink], fields=[a, b])
++- LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])
+   +- LogicalProject(a=[$0], b=[$1])
+      +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+    </Resource>
+    <Resource name="planAfter">
+      <![CDATA[
+Sink(table=[default_catalog.default_database.sink], fields=[a, b])
++- Sort(orderBy=[a ASC])
+   +- Exchange(distribution=[single])
+      +- Calc(select=[a, b])
+         +- BoundedStreamScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c])
+]]>
+    </Resource>
+  </TestCase>
 </Root>
diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala
index fc544da..32a95fb 100644
--- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala
+++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/batch/sql/TableSinkTest.scala
@@ -78,4 +78,22 @@ class TableSinkTest extends TableTestBase {
 
     util.verifyPlan(stmtSet)
   }
+
+  @Test
+  def testDynamicPartWithOrderBy(): Unit = {
+    util.addTable(
+      s"""
+         |CREATE TABLE sink (
+         |  `a` INT,
+         |  `b` BIGINT
+         |) PARTITIONED BY (
+         |  `b`
+         |) WITH (
+         |  'connector' = 'values'
+         |)
+         |""".stripMargin)
+    val stmtSet = util.tableEnv.createStatementSet()
+    stmtSet.addInsertSql("INSERT INTO sink SELECT a,b FROM MyTable ORDER BY a")
+    util.verifyPlan(stmtSet)
+  }
 }