You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ro...@apache.org on 2023/12/27 14:43:22 UTC
(flink) branch master updated: [FLINK-27851][table-planner] Support to access source pks through MiniBatchAssigner
This is an automated email from the ASF dual-hosted git repository.
ron pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new dd16a4c07b2 [FLINK-27851][table-planner] Support to access source pks through MiniBatchAssigner
dd16a4c07b2 is described below
commit dd16a4c07b2f8c96740fb522cb54cfd1d5a5e835
Author: xuyang <xy...@163.com>
AuthorDate: Tue May 31 16:31:37 2022 +0800
[FLINK-27851][table-planner] Support to access source pks through MiniBatchAssigner
This closes #19850
---
.../plan/metadata/FlinkRelMdUniqueKeys.scala | 8 ++++
.../plan/metadata/FlinkRelMdUpsertKeys.scala | 10 ++++-
.../planner/plan/stream/sql/join/JoinTest.xml | 33 ++++++++++++++
.../plan/metadata/FlinkRelMdHandlerTestBase.scala | 3 ++
.../plan/metadata/FlinkRelMdUniqueKeysTest.scala | 5 +++
.../plan/metadata/FlinkRelMdUpsertKeysTest.scala | 5 +++
.../planner/plan/stream/sql/join/JoinTest.scala | 50 ++++++++++++++++++++++
7 files changed, 112 insertions(+), 2 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
index ad63bd58e76..e171925351d 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeys.scala
@@ -112,6 +112,14 @@ class FlinkRelMdUniqueKeys private extends MetadataHandler[BuiltInMetadata.Uniqu
getProjectUniqueKeys(projects, input, mq, ignoreNulls)
}
+ def getUniqueKeys(
+ rel: StreamPhysicalMiniBatchAssigner,
+ mq: RelMetadataQuery,
+ ignoreNulls: Boolean): JSet[ImmutableBitSet] = {
+
+ mq.getUniqueKeys(rel.getInput, ignoreNulls)
+ }
+
private def getProjectUniqueKeys(
projects: JList[RexNode],
input: RelNode,
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
index 386dea8c38b..358862826a2 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeys.scala
@@ -22,7 +22,7 @@ import org.apache.flink.table.planner.plan.metadata.FlinkMetadata.UpsertKeys
import org.apache.flink.table.planner.plan.nodes.calcite.{Expand, Rank, WatermarkAssigner, WindowAggregate}
import org.apache.flink.table.planner.plan.nodes.physical.batch.{BatchPhysicalGroupAggregateBase, BatchPhysicalOverAggregate, BatchPhysicalWindowAggregateBase}
import org.apache.flink.table.planner.plan.nodes.physical.common.CommonPhysicalLookupJoin
-import org.apache.flink.table.planner.plan.nodes.physical.stream.{StreamPhysicalChangelogNormalize, StreamPhysicalDeduplicate, StreamPhysicalDropUpdateBefore, StreamPhysicalGlobalGroupAggregate, StreamPhysicalGroupAggregate, StreamPhysicalGroupWindowAggregate, StreamPhysicalIntervalJoin, StreamPhysicalLocalGroupAggregate, StreamPhysicalOverAggregate}
+import org.apache.flink.table.planner.plan.nodes.physical.stream._
import org.apache.flink.table.planner.plan.schema.IntermediateRelTable
import org.apache.flink.table.planner.plan.utils.FlinkRexUtil
@@ -30,7 +30,7 @@ import com.google.common.collect.ImmutableSet
import org.apache.calcite.plan.hep.HepRelVertex
import org.apache.calcite.plan.volcano.RelSubset
import org.apache.calcite.rel.{RelDistribution, RelNode, SingleRel}
-import org.apache.calcite.rel.core.{Aggregate, Calc, Exchange, Filter, Join, JoinInfo, JoinRelType, Project, SetOp, Sort, TableScan, Window}
+import org.apache.calcite.rel.core._
import org.apache.calcite.rel.metadata._
import org.apache.calcite.rex.{RexNode, RexUtil}
import org.apache.calcite.util.{Bug, ImmutableBitSet, Util}
@@ -114,6 +114,12 @@ class FlinkRelMdUpsertKeys private extends MetadataHandler[UpsertKeys] {
ImmutableSet.of(ImmutableBitSet.of(rel.uniqueKeys.map(Integer.valueOf).toList))
}
+ def getUpsertKeys(
+ rel: StreamPhysicalMiniBatchAssigner,
+ mq: RelMetadataQuery): JSet[ImmutableBitSet] = {
+ FlinkRelMetadataQuery.reuseOrCreate(mq).getUpsertKeys(rel.getInput)
+ }
+
def getUpsertKeys(
rel: StreamPhysicalDropUpdateBefore,
mq: RelMetadataQuery): JSet[ImmutableBitSet] = {
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml
index 2ab56e29378..1a4d0b5e002 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml
@@ -576,6 +576,39 @@ Calc(select=[a1])
: +- LegacyTableSourceScan(table=[[default_catalog, default_database, A, source: [TestTableSource(a1, a2, a3)]]], fields=[a1, a2, a3])
+- Exchange(distribution=[hash[pk1]])
+- TableSourceScan(table=[[default_catalog, default_database, tableWithCompositePk, project=[pk1], metadata=[]]], fields=[pk1])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testJoinAccessSourcePkWithMiniBatchAssigner">
+ <Resource name="explain">
+ <![CDATA[== Abstract Syntax Tree ==
+LogicalSink(table=[default_catalog.default_database.sink], fields=[a, b, d])
++- LogicalProject(a=[$0], b=[$1], d=[$3])
+ +- LogicalJoin(condition=[=($0, $2)], joinType=[inner])
+ :- LogicalTableScan(table=[[default_catalog, default_database, left_table]])
+ +- LogicalTableScan(table=[[default_catalog, default_database, right_table]])
+
+== Optimized Physical Plan ==
+Sink(table=[default_catalog.default_database.sink], fields=[a, b, d])
++- Calc(select=[a, b, d])
+ +- Join(joinType=[InnerJoin], where=[=(a, c)], select=[a, b, c, d], leftInputSpec=[JoinKeyContainsUniqueKey], rightInputSpec=[JoinKeyContainsUniqueKey])
+ :- Exchange(distribution=[hash[a]])
+ : +- MiniBatchAssigner(interval=[10000ms], mode=[ProcTime])
+ : +- TableSourceScan(table=[[default_catalog, default_database, left_table]], fields=[a, b])
+ +- Exchange(distribution=[hash[c]])
+ +- MiniBatchAssigner(interval=[10000ms], mode=[ProcTime])
+ +- TableSourceScan(table=[[default_catalog, default_database, right_table]], fields=[c, d])
+
+== Optimized Execution Plan ==
+Sink(table=[default_catalog.default_database.sink], fields=[a, b, d])
++- Calc(select=[a, b, d])
+ +- Join(joinType=[InnerJoin], where=[(a = c)], select=[a, b, c, d], leftInputSpec=[JoinKeyContainsUniqueKey], rightInputSpec=[JoinKeyContainsUniqueKey])
+ :- Exchange(distribution=[hash[a]])
+ : +- MiniBatchAssigner(interval=[10000ms], mode=[ProcTime])
+ : +- TableSourceScan(table=[[default_catalog, default_database, left_table]], fields=[a, b])
+ +- Exchange(distribution=[hash[c]])
+ +- MiniBatchAssigner(interval=[10000ms], mode=[ProcTime])
+ +- TableSourceScan(table=[[default_catalog, default_database, right_table]], fields=[c, d])
]]>
</Resource>
</TestCase>
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
index b063f96b404..ff6eed9be7d 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdHandlerTestBase.scala
@@ -310,6 +310,9 @@ class FlinkRelMdHandlerTestBase {
relBuilder.watermark(4, watermarkRexNode).build()
}
+ protected lazy val streamMiniBatchAssigner: RelNode =
+ new StreamPhysicalMiniBatchAssigner(cluster, streamPhysicalTraits, studentStreamScan)
+
// id, name, score, age, height, sex, class, 1
// id, null, score, age, height, sex, class, 4
// id, null, score, age, height, null, class, 5
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeysTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeysTest.scala
index 28d51c8829d..ddb440bc320 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeysTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUniqueKeysTest.scala
@@ -106,6 +106,11 @@ class FlinkRelMdUniqueKeysTest extends FlinkRelMdHandlerTestBase {
assertEquals(uniqueKeys(Array(0)), mq.getUniqueKeys(logicalWatermarkAssigner).toSet)
}
+ @Test
+ def testGetUniqueKeysOnMiniBatchAssigner(): Unit = {
+ assertEquals(uniqueKeys(Array(0)), mq.getUniqueKeys(streamMiniBatchAssigner).toSet)
+ }
+
@Test
def testGetUniqueKeysOnCalc(): Unit = {
relBuilder.push(studentLogicalScan)
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala
index 363f04b3371..40e38dda161 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdUpsertKeysTest.scala
@@ -106,6 +106,11 @@ class FlinkRelMdUpsertKeysTest extends FlinkRelMdHandlerTestBase {
assertEquals(toBitSet(Array(0)), mq.getUpsertKeys(logicalWatermarkAssigner).toSet)
}
+ @Test
+ def testGetUpsertKeysOnMiniBatchAssigner(): Unit = {
+ assertEquals(toBitSet(Array(0)), mq.getUpsertKeys(streamMiniBatchAssigner).toSet)
+ }
+
@Test
def testGetUpsertKeysOnCalc(): Unit = {
relBuilder.push(studentLogicalScan)
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala
index 4df4c3de8dd..4f50c1b3d20 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.scala
@@ -19,10 +19,13 @@ package org.apache.flink.table.planner.plan.stream.sql.join
import org.apache.flink.api.scala._
import org.apache.flink.table.api._
+import org.apache.flink.table.api.config.ExecutionConfigOptions
import org.apache.flink.table.planner.utils.{StreamTableTestUtil, TableFunc1, TableTestBase}
import org.junit.jupiter.api.Test
+import java.time.Duration
+
class JoinTest extends TableTestBase {
private val util: StreamTableTestUtil = streamTestUtil()
@@ -612,4 +615,51 @@ class JoinTest extends TableTestBase {
|SELECT * FROM leftPartitionTable, rightPartitionTable WHERE b1 = 1 AND b2 = 3 AND a1 = a2
|""".stripMargin)
}
+
+ @Test
+ def testJoinAccessSourcePkWithMiniBatchAssigner(): Unit = {
+ util.tableEnv.executeSql("""
+ |create table left_table (
+ | a varchar primary key not enforced,
+ | b int
+ |) with (
+ | 'connector' = 'values'
+ |)
+ |""".stripMargin)
+
+ util.tableEnv.executeSql("""
+ |create table right_table (
+ | c varchar primary key not enforced,
+ | d int
+ |) with (
+ | 'connector' = 'values'
+ |)
+ |""".stripMargin)
+
+ util.tableEnv.executeSql("""
+ |create table sink (
+ | e varchar primary key not enforced,
+ | f int,
+ | g int
+ |) with (
+ | 'connector' = 'values'
+ |)
+ |""".stripMargin)
+
+ util.tableEnv.getConfig.getConfiguration
+ .set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ENABLED, Boolean.box(true))
+ util.tableEnv.getConfig.getConfiguration
+ .set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ALLOW_LATENCY, Duration.ofSeconds(10))
+ util.tableEnv.getConfig.getConfiguration
+ .set(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_SIZE, Long.box(5000L))
+
+ util.verifyExplainInsert(
+ """
+ |insert into sink
+ |select left_table.a, left_table.b, right_table.d
+ | from left_table
+ | join right_table on left_table.a = right_table.c
+ |""".stripMargin
+ )
+ }
}