You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ro...@apache.org on 2023/12/27 14:43:22 UTC

(flink) branch master updated: [FLINK-27851][table-planner] Support to access source pks through MiniBatchAssigner

This is an automated email from the ASF dual-hosted git repository.

ron pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new dd16a4c07b2 [FLINK-27851][table-planner] Support to access source pks through MiniBatchAssigner
dd16a4c07b2 is described below

commit dd16a4c07b2f8c96740fb522cb54cfd1d5a5e835
Author: xuyang <xy...@163.com>
AuthorDate: Tue May 31 16:31:37 2022 +0800

    [FLINK-27851][table-planner] Support to access source pks through MiniBatchAssigner
    
    This closes #19850
---
 .../plan/metadata/FlinkRelMdUniqueKeys.scala       |  8 ++++
 .../plan/metadata/FlinkRelMdUpsertKeys.scala       | 10 ++++-
 .../planner/plan/stream/sql/join/JoinTest.xml      | 33 ++++++++++++++
 .../plan/metadata/FlinkRelMdHandlerTestBase.scala  |  3 ++
 .../plan/metadata/FlinkRelMdUniqueKeysTest.scala   |  5 +++
 .../plan/metadata/FlinkRelMdUpsertKeysTest.scala   |  5 +++
 .../planner/plan/stream/sql/join/JoinTest.scala    | 50 ++++++++++++++++++++++
 7 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
index ad63bd58e76..e171925351d 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
@@ -112,6 +112,14 @@ class FlinkRelMdUniqueKeys private extends MetadataHandler[BuiltInMetadata.Uniqu
     getProjectUniqueKeys(projects, input, mq, ignoreNulls)
   }
 
+  def getUniqueKeys(
+      rel: StreamPhysicalMiniBatchAssigner,
+      mq: RelMetadataQuery,
+      ignoreNulls: Boolean): JSet[ImmutableBitSet] = {
+
+    mq.getUniqueKeys(rel.getInput, ignoreNulls)
+  }
+
   private def getProjectUniqueKeys(
       projects: JList[RexNode],
       input: RelNode,
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
index 386dea8c38b..358862826a2 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
@@ -22,7 +22,7 @@ import org.apache.flink.table.planner.plan.metadata.FlinkMetadata.UpsertKeys
 import org.apache.flink.table.planner.plan.nodes.calcite.{Expand, Rank, WatermarkAssigner, WindowAggregate}
 import org.apache.flink.table.planner.plan.nodes.physical.batch.{BatchPhysicalGroupAggregateBase, BatchPhysicalOverAggregate, BatchPhysicalWindowAggregateBase}
 import org.apache.flink.table.planner.plan.nodes.physical.common.CommonPhysicalLookupJoin
-import org.apache.flink.table.planner.plan.nodes.physical.stream.{StreamPhysicalChangelogNormalize, StreamPhysicalDeduplicate, StreamPhysicalDropUpdateBefore, StreamPhysicalGlobalGroupAggregate, StreamPhysicalGroupAggregate, StreamPhysicalGroupWindowAggregate, StreamPhysicalIntervalJoin, StreamPhysicalLocalGroupAggregate, StreamPhysicalOverAggregate}
+import org.apache.flink.table.planner.plan.nodes.physical.stream._
 import org.apache.flink.table.planner.plan.schema.IntermediateRelTable
 import org.apache.flink.table.planner.plan.utils.FlinkRexUtil
 
@@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableSet
 import org.apache.calcite.plan.hep.HepRelVertex
 import org.apache.calcite.plan.volcano.RelSubset
 import org.apache.calcite.rel.{RelDistribution, RelNode, SingleRel}
-import org.apache.calcite.rel.core.{Aggregate, Calc, Exchange, Filter, Join, JoinInfo, JoinRelType, Project, SetOp, Sort, TableScan, Window}
+import org.apache.calcite.rel.core._
 import org.apache.calcite.rel.metadata._
 import org.apache.calcite.rex.{RexNode, RexUtil}
 import org.apache.calcite.util.{Bug, ImmutableBitSet, Util}
@@ -114,6 +114,12 @@ class FlinkRelMdUpsertKeys private extends MetadataHandler[UpsertKeys] {
     ImmutableSet.of(ImmutableBitSet.of(rel.uniqueKeys.map(Integer.valueOf).toList))
   }
 
+  def getUpsertKeys(
+      rel: StreamPhysicalMiniBatchAssigner,
+      mq: RelMetadataQuery): JSet[ImmutableBitSet] = {
+    FlinkRelMetadataQuery.reuseOrCreate(mq).getUpsertKeys(rel.getInput)
+  }
+
   def getUpsertKeys(
       rel: StreamPhysicalDropUpdateBefore,
       mq: RelMetadataQuery): JSet[ImmutableBitSet] = {
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml
index 2ab56e29378..1a4d0b5e002 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml
@@ -576,6 +576,39 @@ Calc(select=[a1])
    :     +- LegacyTableSourceScan(table=[[default_catalog, default_database, A, source: [TestTableSource(a1, a2, a3)]]], fields=[a1, a2, a3])
    +- Exchange(distribution=[hash[pk1]])
       +- TableSourceScan(table=[[default_catalog, default_database, tableWithCompositePk, project=[pk1], metadata=[]]], fields=[pk1])
+]]>
+    </Resource>
+  </TestCase>
+  <TestCase name="testJoinAccessSourcePkWithMiniBatchAssigner">
+    <Resource name="explain">
+      <![CDATA[== Abstract Syntax Tree ==
+LogicalSink(table=[default_catalog.default_database.sink], fields=[a, b, d])
++- LogicalProject(a=[$0], b=[$1], d=[$3])
+   +- LogicalJoin(condition=[=($0, $2)], joinType=[inner])
+      :- LogicalTableScan(table=[[default_catalog, default_database, left_table]])
+      +- LogicalTableScan(table=[[default_catalog, default_database, right_table]])
+
+== Optimized Physical Plan ==
+Sink(table=[default_catalog.default_database.sink], fields=[a, b, d])
++- Calc(select=[a, b, d])
+   +- Join(joinType=[InnerJoin], where=[=(a, c)], select=[a, b, c, d], leftInputSpec=[JoinKeyContainsUniqueKey], rightInputSpec=[JoinKeyContainsUniqueKey])
+      :- Exchange(distribution=[hash[a]])
+      :  +- MiniBatchAssigner(interval=[10000ms], mode=[ProcTime])
+      :     +- TableSourceScan(table=[[default_catalog, default_database, left_table]], fields=[a, b])
+      +- Exchange(distribution=[hash[c]])
+         +- MiniBatchAssigner(interval=[10000ms], mode=[ProcTime])
+            +- TableSourceScan(table=[[default_catalog, default_database, right_table]], fields=[c, d])
+
+== Optimized Execution Plan ==
+Sink(table=[default_catalog.default_database.sink], fields=[a, b, d])
++- Calc(select=[a, b, d])
+   +- Join(joinType=[InnerJoin], where=[(a = c)], select=[a, b, c, d], leftInputSpec=[JoinKeyContainsUniqueKey], rightInputSpec=[JoinKeyContainsUniqueKey])
+      :- Exchange(distribution=[hash[a]])
+      :  +- MiniBatchAssigner(interval=[10000ms], mode=[ProcTime])
+      :     +- TableSourceScan(table=[[default_catalog, default_database, left_table]], fields=[a, b])
+      +- Exchange(distribution=[hash[c]])
+         +- MiniBatchAssigner(interval=[10000ms], mode=[ProcTime])
+            +- TableSourceScan(table=[[default_catalog, default_database, right_table]], fields=[c, d])
 ]]>
     </Resource>
   </TestCase>
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
index b063f96b404..ff6eed9be7d 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
@@ -310,6 +310,9 @@ class FlinkRelMdHandlerTestBase {
     relBuilder.watermark(4, watermarkRexNode).build()
   }
 
+  protected lazy val streamMiniBatchAssigner: RelNode =
+    new StreamPhysicalMiniBatchAssigner(cluster, streamPhysicalTraits, studentStreamScan)
+
   // id, name, score, age, height, sex, class, 1
   // id, null, score, age, height, sex, class, 4
   // id, null, score, age, height, null, class, 5
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeysTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeysTest.scala
index 28d51c8829d..ddb440bc320 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeysTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeysTest.scala
@@ -106,6 +106,11 @@ class FlinkRelMdUniqueKeysTest extends FlinkRelMdHandlerTestBase {
     assertEquals(uniqueKeys(Array(0)), mq.getUniqueKeys(logicalWatermarkAssigner).toSet)
   }
 
+  @Test
+  def testGetUniqueKeysOnMiniBatchAssigner(): Unit = {
+    assertEquals(uniqueKeys(Array(0)), mq.getUniqueKeys(streamMiniBatchAssigner).toSet)
+  }
+
   @Test
   def testGetUniqueKeysOnCalc(): Unit = {
     relBuilder.push(studentLogicalScan)
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala
index 363f04b3371..40e38dda161 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala
@@ -106,6 +106,11 @@ class FlinkRelMdUpsertKeysTest extends FlinkRelMdHandlerTestBase {
     assertEquals(toBitSet(Array(0)), mq.getUpsertKeys(logicalWatermarkAssigner).toSet)
   }
 
+  @Test
+  def testGetUpsertKeysOnMiniBatchAssigner(): Unit = {
+    assertEquals(toBitSet(Array(0)), mq.getUpsertKeys(streamMiniBatchAssigner).toSet)
+  }
+
   @Test
   def testGetUpsertKeysOnCalc(): Unit = {
     relBuilder.push(studentLogicalScan)
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala
index 4df4c3de8dd..4f50c1b3d20 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala
@@ -19,10 +19,13 @@ package org.apache.flink.table.planner.plan.stream.sql.join
 
 import org.apache.flink.api.scala._
 import org.apache.flink.table.api._
+import org.apache.flink.table.api.config.ExecutionConfigOptions
 import org.apache.flink.table.planner.utils.{StreamTableTestUtil, TableFunc1, TableTestBase}
 
 import org.junit.jupiter.api.Test
 
+import java.time.Duration
+
 class JoinTest extends TableTestBase {
 
   private val util: StreamTableTestUtil = streamTestUtil()
@@ -612,4 +615,51 @@ class JoinTest extends TableTestBase {
         |SELECT * FROM leftPartitionTable, rightPartitionTable WHERE b1 = 1 AND b2 = 3 AND a1 = a2
         |""".stripMargin)
   }
+
+  @Test
+  def testJoinAccessSourcePkWithMiniBatchAssigner(): Unit = {
+    util.tableEnv.executeSql("""
+                               |create table left_table (
+                               | a varchar primary key not enforced,
+                               | b int
+                               |) with (
+                               | 'connector' = 'values'
+                               |)
+                               |""".stripMargin)
+
+    util.tableEnv.executeSql("""
+                               |create table right_table (
+                               | c varchar primary key not enforced,
+                               | d int
+                               |) with (
+                               | 'connector' = 'values'
+                               |)
+                               |""".stripMargin)
+
+    util.tableEnv.executeSql("""
+                               |create table sink (
+                               | e varchar primary key not enforced,
+                               | f int,
+                               | g int
+                               |) with (
+                               | 'connector' = 'values'
+                               |)
+                               |""".stripMargin)
+
+    util.tableEnv.getConfig.getConfiguration
+      .set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ENABLED, Boolean.box(true))
+    util.tableEnv.getConfig.getConfiguration
+      .set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ALLOW_LATENCY, Duration.ofSeconds(10))
+    util.tableEnv.getConfig.getConfiguration
+      .set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_SIZE, Long.box(5000L))
+
+    util.verifyExplainInsert(
+      """
+        |insert into sink
+        |select left_table.a, left_table.b, right_table.d
+        | from left_table
+        | join right_table on left_table.a = right_table.c
+        |""".stripMargin
+    )
+  }
 }