You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sn...@apache.org on 2025/06/30 06:38:58 UTC
(flink) branch master updated: [FLINK-38027][table] Simplify watermark expressions
This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 2608c44b7d1 [FLINK-38027][table] Simplify watermark expressions
2608c44b7d1 is described below
commit 2608c44b7d189b4dfa11f7026a9e5145b266354e
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Mon Jun 30 08:38:52 2025 +0200
[FLINK-38027][table] Simplify watermark expressions
---
.../nodes/calcite/LogicalWatermarkAssigner.scala | 8 +++-
.../plan/nodes/calcite/WatermarkAssigner.scala | 13 +++++-
.../logical/FlinkLogicalWatermarkAssigner.scala | 6 +--
.../stream/StreamPhysicalWatermarkAssigner.scala | 6 +--
.../plan/stream/sql/SourceWatermarkTest.xml | 54 ++++++++++++++++++++++
.../plan/stream/sql/SourceWatermarkTest.scala | 34 +++++++++++++-
6 files changed, 111 insertions(+), 10 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalWatermarkAssigner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalWatermarkAssigner.scala
index c9d3443f526..f27d59f4949 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalWatermarkAssigner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalWatermarkAssigner.scala
@@ -61,6 +61,12 @@ object LogicalWatermarkAssigner {
rowtimeFieldIndex: Int,
watermarkExpr: RexNode): LogicalWatermarkAssigner = {
val traits = cluster.traitSetOf(Convention.NONE)
- new LogicalWatermarkAssigner(cluster, traits, input, hints, rowtimeFieldIndex, watermarkExpr)
+ new LogicalWatermarkAssigner(
+ cluster,
+ traits,
+ input,
+ hints,
+ rowtimeFieldIndex,
+ WatermarkUtils.simplify(cluster, watermarkExpr))
}
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/WatermarkAssigner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/WatermarkAssigner.scala
index 58137c195ad..c308c4110b2 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/WatermarkAssigner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/WatermarkAssigner.scala
@@ -20,11 +20,11 @@ package org.apache.flink.table.planner.plan.nodes.calcite
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
import com.google.common.collect.ImmutableList
-import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
+import org.apache.calcite.plan.{RelOptCluster, RelOptPredicateList, RelTraitSet}
import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFieldImpl}
import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
import org.apache.calcite.rel.hint.{Hintable, RelHint}
-import org.apache.calcite.rex.RexNode
+import org.apache.calcite.rex.{RexNode, RexSimplify}
import org.apache.calcite.sql.`type`.SqlTypeName
import java.util
@@ -91,3 +91,12 @@ abstract class WatermarkAssigner(
def withHints(hintList: util.List[RelHint]): RelNode
}
+
+object WatermarkUtils {
+ def simplify(cluster: RelOptCluster, watermarkExpr: RexNode): RexNode = {
+ new RexSimplify(
+ cluster.getRexBuilder,
+ RelOptPredicateList.EMPTY,
+ cluster.getPlanner.getExecutor).simplify(watermarkExpr)
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalWatermarkAssigner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalWatermarkAssigner.scala
index 7468c83142f..65e5de3969a 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalWatermarkAssigner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalWatermarkAssigner.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.planner.plan.nodes.logical
import org.apache.flink.table.planner.plan.nodes.FlinkConventions
-import org.apache.flink.table.planner.plan.nodes.calcite.{LogicalWatermarkAssigner, WatermarkAssigner}
+import org.apache.flink.table.planner.plan.nodes.calcite.{LogicalWatermarkAssigner, WatermarkAssigner, WatermarkUtils}
import org.apache.calcite.plan._
import org.apache.calcite.rel.RelNode
@@ -61,7 +61,7 @@ class FlinkLogicalWatermarkAssigner(
input,
hints,
rowtimeFieldIndex,
- watermarkExpr)
+ WatermarkUtils.simplify(cluster, watermarkExpr))
}
}
@@ -97,6 +97,6 @@ object FlinkLogicalWatermarkAssigner {
input,
Collections.emptyList(),
rowtimeFieldIndex,
- watermarkExpr)
+ WatermarkUtils.simplify(cluster, watermarkExpr))
}
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWatermarkAssigner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWatermarkAssigner.scala
index f9c7ebb6861..bf343773bd3 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWatermarkAssigner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWatermarkAssigner.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.planner.plan.nodes.physical.stream
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
-import org.apache.flink.table.planner.plan.nodes.calcite.WatermarkAssigner
+import org.apache.flink.table.planner.plan.nodes.calcite.{WatermarkAssigner, WatermarkUtils}
import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty}
import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecWatermarkAssigner
import org.apache.flink.table.planner.plan.utils.RelExplainUtil.preferExpressionFormat
@@ -74,7 +74,7 @@ class StreamPhysicalWatermarkAssigner(
override def translateToExecNode(): ExecNode[_] = {
new StreamExecWatermarkAssigner(
unwrapTableConfig(this),
- watermarkExpr,
+ WatermarkUtils.simplify(cluster, watermarkExpr),
rowtimeFieldIndex,
InputProperty.DEFAULT,
FlinkTypeFactory.toLogicalRowType(getRowType),
@@ -88,6 +88,6 @@ class StreamPhysicalWatermarkAssigner(
input,
hints,
rowtimeFieldIndex,
- watermarkExpr)
+ WatermarkUtils.simplify(cluster, watermarkExpr))
}
}
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.xml
index 5218853ce1f..cc48573fe55 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.xml
@@ -50,6 +50,60 @@ LogicalProject(a=[$0], b=[$1], ts=[$5])
<Resource name="optimized exec plan">
<![CDATA[
TableSourceScan(table=[[default_catalog, default_database, t1, project=[a, b, t], metadata=[], watermark=[-(t, 10000:INTERVAL SECOND)], watermarkEmitStrategy=[on-periodic]]], fields=[a, b, t])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testProjectTransposeWatermarkAssignerWithSimplifiableWatermarks[[1] watermarkExp=`ts` - ( - ( - (INTERVAL '10' SECOND)))]">
+ <Resource name="sql">
+ <![CDATA[SELECT a, b, ts FROM t1]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(a=[$0], b=[$1], ts=[$5])
++- LogicalWatermarkAssigner(rowtime=[ts], watermark=[-($5, 10000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], t=[$4], ts=[$4])
+ +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
+]]>
+ </Resource>
+ <Resource name="optimized exec plan">
+ <![CDATA[
+TableSourceScan(table=[[default_catalog, default_database, t1, project=[a, b, t], metadata=[], watermark=[-(t, 10000:INTERVAL SECOND)], watermarkEmitStrategy=[on-periodic]]], fields=[a, b, t])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testProjectTransposeWatermarkAssignerWithSimplifiableWatermarks[[2] watermarkExp=`ts` - ARRAY[INTERVAL '10' SECOND, INTERVAL '1' SECOND][1]]">
+ <Resource name="sql">
+ <![CDATA[SELECT a, b, ts FROM t1]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(a=[$0], b=[$1], ts=[$5])
++- LogicalWatermarkAssigner(rowtime=[ts], watermark=[-($5, ITEM(ARRAY(10000:INTERVAL SECOND, 1000:INTERVAL SECOND), 1))])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], t=[$4], ts=[$4])
+ +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
+]]>
+ </Resource>
+ <Resource name="optimized exec plan">
+ <![CDATA[
+TableSourceScan(table=[[default_catalog, default_database, t1, project=[a, b, t], metadata=[], watermark=[-(t, ITEM(ARRAY(10000:INTERVAL SECOND, 1000:INTERVAL SECOND), 1))], watermarkEmitStrategy=[on-periodic]]], fields=[a, b, t])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testProjectTransposeWatermarkAssignerWithSimplifiableWatermarks[[3] watermarkExp=`ts` - CASE WHEN true THEN INTERVAL '10' SECOND ELSE INTERVAL '2' SECOND END]">
+ <Resource name="sql">
+ <![CDATA[SELECT a, b, ts FROM t1]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(a=[$0], b=[$1], ts=[$5])
++- LogicalWatermarkAssigner(rowtime=[ts], watermark=[-($5, 10000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], t=[$4], ts=[$4])
+ +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
+]]>
+ </Resource>
+ <Resource name="optimized exec plan">
+ <![CDATA[
+TableSourceScan(table=[[default_catalog, default_database, t1, project=[a, b, t], metadata=[], watermark=[-(t, 10000:INTERVAL SECOND)], watermarkEmitStrategy=[on-periodic]]], fields=[a, b, t])
]]>
</Resource>
</TestCase>
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.scala
index 1fc524d4ae1..913d8f21482 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.scala
@@ -21,6 +21,8 @@ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctio
import org.apache.flink.table.planner.utils.TableTestBase
import org.junit.jupiter.api.{BeforeEach, Test}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ValueSource
/** Tests for watermark push down. */
class SourceWatermarkTest extends TableTestBase {
@@ -179,7 +181,37 @@ class SourceWatermarkTest extends TableTestBase {
| `d` INT,
| `t` TIMESTAMP(3),
| `ts` AS `t`,
- | WATERMARK FOR `ts` AS `ts` - INTERVAL '10' SECOND
+ | WATERMARK FOR `ts` AS `ts` - INTERVAL '10' SECOND
+ |) WITH (
+ | 'connector' = 'values',
+ | 'enable-watermark-push-down' = 'true',
+ | 'bounded' = 'false',
+ | 'disable-lookup' = 'true'
+ |)
+ """.stripMargin
+ util.tableEnv.executeSql(sourceDDL)
+ util.verifyExecPlan("SELECT a, b, ts FROM t1")
+ }
+
+ @ParameterizedTest
+ @ValueSource(
+ strings = Array[String](
+ "`ts` - ( - ( - (INTERVAL '10' SECOND)))",
+ "`ts` - ARRAY[INTERVAL '10' SECOND, INTERVAL '1' SECOND][1]",
+ "`ts` - CASE WHEN true THEN INTERVAL '10' SECOND ELSE INTERVAL '2' SECOND END"
+ ))
+ def testProjectTransposeWatermarkAssignerWithSimplifiableWatermarks(
+ watermarkExp: String): Unit = {
+ val sourceDDL =
+ s"""
+ |CREATE TEMPORARY TABLE `t1` (
+ | `a` VARCHAR,
+ | `b` VARCHAR,
+ | `c` VARCHAR,
+ | `d` INT,
+ | `t` TIMESTAMP(3),
+ | `ts` AS `t`,
+ | WATERMARK FOR `ts` AS $watermarkExp
|) WITH (
| 'connector' = 'values',
| 'enable-watermark-push-down' = 'true',