You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sn...@apache.org on 2025/06/30 06:38:58 UTC

(flink) branch master updated: [FLINK-38027][table] Simplify watermark expressions

This is an automated email from the ASF dual-hosted git repository.

snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 2608c44b7d1 [FLINK-38027][table] Simplify watermark expressions
2608c44b7d1 is described below

commit 2608c44b7d189b4dfa11f7026a9e5145b266354e
Author: Sergey Nuyanzin <sn...@gmail.com>
AuthorDate: Mon Jun 30 08:38:52 2025 +0200

    [FLINK-38027][table] Simplify watermark expressions
---
 .../nodes/calcite/LogicalWatermarkAssigner.scala   |  8 +++-
 .../plan/nodes/calcite/WatermarkAssigner.scala     | 13 +++++-
 .../logical/FlinkLogicalWatermarkAssigner.scala    |  6 +--
 .../stream/StreamPhysicalWatermarkAssigner.scala   |  6 +--
 .../plan/stream/sql/SourceWatermarkTest.xml        | 54 ++++++++++++++++++++++
 .../plan/stream/sql/SourceWatermarkTest.scala      | 34 +++++++++++++-
 6 files changed, 111 insertions(+), 10 deletions(-)

diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalWatermarkAssigner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalWatermarkAssigner.scala
index c9d3443f526..f27d59f4949 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalWatermarkAssigner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/LogicalWatermarkAssigner.scala
@@ -61,6 +61,12 @@ object LogicalWatermarkAssigner {
       rowtimeFieldIndex: Int,
       watermarkExpr: RexNode): LogicalWatermarkAssigner = {
     val traits = cluster.traitSetOf(Convention.NONE)
-    new LogicalWatermarkAssigner(cluster, traits, input, hints, rowtimeFieldIndex, watermarkExpr)
+    new LogicalWatermarkAssigner(
+      cluster,
+      traits,
+      input,
+      hints,
+      rowtimeFieldIndex,
+      WatermarkUtils.simplify(cluster, watermarkExpr))
   }
 }
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/WatermarkAssigner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/WatermarkAssigner.scala
index 58137c195ad..c308c4110b2 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/WatermarkAssigner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/calcite/WatermarkAssigner.scala
@@ -20,11 +20,11 @@ package org.apache.flink.table.planner.plan.nodes.calcite
 import org.apache.flink.table.planner.calcite.FlinkTypeFactory
 
 import com.google.common.collect.ImmutableList
-import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
+import org.apache.calcite.plan.{RelOptCluster, RelOptPredicateList, RelTraitSet}
 import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFieldImpl}
 import org.apache.calcite.rel.{RelNode, RelWriter, SingleRel}
 import org.apache.calcite.rel.hint.{Hintable, RelHint}
-import org.apache.calcite.rex.RexNode
+import org.apache.calcite.rex.{RexNode, RexSimplify}
 import org.apache.calcite.sql.`type`.SqlTypeName
 
 import java.util
@@ -91,3 +91,12 @@ abstract class WatermarkAssigner(
 
   def withHints(hintList: util.List[RelHint]): RelNode
 }
+
+object WatermarkUtils {
+  def simplify(cluster: RelOptCluster, watermarkExpr: RexNode): RexNode = {
+    new RexSimplify(
+      cluster.getRexBuilder,
+      RelOptPredicateList.EMPTY,
+      cluster.getPlanner.getExecutor).simplify(watermarkExpr)
+  }
+}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalWatermarkAssigner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalWatermarkAssigner.scala
index 7468c83142f..65e5de3969a 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalWatermarkAssigner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalWatermarkAssigner.scala
@@ -18,7 +18,7 @@
 package org.apache.flink.table.planner.plan.nodes.logical
 
 import org.apache.flink.table.planner.plan.nodes.FlinkConventions
-import org.apache.flink.table.planner.plan.nodes.calcite.{LogicalWatermarkAssigner, WatermarkAssigner}
+import org.apache.flink.table.planner.plan.nodes.calcite.{LogicalWatermarkAssigner, WatermarkAssigner, WatermarkUtils}
 
 import org.apache.calcite.plan._
 import org.apache.calcite.rel.RelNode
@@ -61,7 +61,7 @@ class FlinkLogicalWatermarkAssigner(
       input,
       hints,
       rowtimeFieldIndex,
-      watermarkExpr)
+      WatermarkUtils.simplify(cluster, watermarkExpr))
   }
 }
 
@@ -97,6 +97,6 @@ object FlinkLogicalWatermarkAssigner {
       input,
       Collections.emptyList(),
       rowtimeFieldIndex,
-      watermarkExpr)
+      WatermarkUtils.simplify(cluster, watermarkExpr))
   }
 }
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWatermarkAssigner.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWatermarkAssigner.scala
index f9c7ebb6861..bf343773bd3 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWatermarkAssigner.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalWatermarkAssigner.scala
@@ -18,7 +18,7 @@
 package org.apache.flink.table.planner.plan.nodes.physical.stream
 
 import org.apache.flink.table.planner.calcite.FlinkTypeFactory
-import org.apache.flink.table.planner.plan.nodes.calcite.WatermarkAssigner
+import org.apache.flink.table.planner.plan.nodes.calcite.{WatermarkAssigner, WatermarkUtils}
 import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty}
 import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecWatermarkAssigner
 import org.apache.flink.table.planner.plan.utils.RelExplainUtil.preferExpressionFormat
@@ -74,7 +74,7 @@ class StreamPhysicalWatermarkAssigner(
   override def translateToExecNode(): ExecNode[_] = {
     new StreamExecWatermarkAssigner(
       unwrapTableConfig(this),
-      watermarkExpr,
+      WatermarkUtils.simplify(cluster, watermarkExpr),
       rowtimeFieldIndex,
       InputProperty.DEFAULT,
       FlinkTypeFactory.toLogicalRowType(getRowType),
@@ -88,6 +88,6 @@ class StreamPhysicalWatermarkAssigner(
       input,
       hints,
       rowtimeFieldIndex,
-      watermarkExpr)
+      WatermarkUtils.simplify(cluster, watermarkExpr))
   }
 }
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.xml
index 5218853ce1f..cc48573fe55 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.xml
@@ -50,6 +50,60 @@ LogicalProject(a=[$0], b=[$1], ts=[$5])
     <Resource name="optimized exec plan">
       <![CDATA[
 TableSourceScan(table=[[default_catalog, default_database, t1, project=[a, b, t], metadata=[], watermark=[-(t, 10000:INTERVAL SECOND)], watermarkEmitStrategy=[on-periodic]]], fields=[a, b, t])
+]]>
+    </Resource>
+  </TestCase>
+  <TestCase name="testProjectTransposeWatermarkAssignerWithSimplifiableWatermarks[[1] watermarkExp=`ts` - ( - ( - (INTERVAL &#39;10&#39; SECOND)))]">
+    <Resource name="sql">
+      <![CDATA[SELECT a, b, ts FROM t1]]>
+    </Resource>
+    <Resource name="ast">
+      <![CDATA[
+LogicalProject(a=[$0], b=[$1], ts=[$5])
++- LogicalWatermarkAssigner(rowtime=[ts], watermark=[-($5, 10000:INTERVAL SECOND)])
+   +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], t=[$4], ts=[$4])
+      +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
+]]>
+    </Resource>
+    <Resource name="optimized exec plan">
+      <![CDATA[
+TableSourceScan(table=[[default_catalog, default_database, t1, project=[a, b, t], metadata=[], watermark=[-(t, 10000:INTERVAL SECOND)], watermarkEmitStrategy=[on-periodic]]], fields=[a, b, t])
+]]>
+    </Resource>
+  </TestCase>
+  <TestCase name="testProjectTransposeWatermarkAssignerWithSimplifiableWatermarks[[2] watermarkExp=`ts` - ARRAY[INTERVAL &#39;10&#39; SECOND, INTERVAL &#39;1&#39; SECOND][1]]">
+    <Resource name="sql">
+      <![CDATA[SELECT a, b, ts FROM t1]]>
+    </Resource>
+    <Resource name="ast">
+      <![CDATA[
+LogicalProject(a=[$0], b=[$1], ts=[$5])
++- LogicalWatermarkAssigner(rowtime=[ts], watermark=[-($5, ITEM(ARRAY(10000:INTERVAL SECOND, 1000:INTERVAL SECOND), 1))])
+   +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], t=[$4], ts=[$4])
+      +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
+]]>
+    </Resource>
+    <Resource name="optimized exec plan">
+      <![CDATA[
+TableSourceScan(table=[[default_catalog, default_database, t1, project=[a, b, t], metadata=[], watermark=[-(t, ITEM(ARRAY(10000:INTERVAL SECOND, 1000:INTERVAL SECOND), 1))], watermarkEmitStrategy=[on-periodic]]], fields=[a, b, t])
+]]>
+    </Resource>
+  </TestCase>
+  <TestCase name="testProjectTransposeWatermarkAssignerWithSimplifiableWatermarks[[3] watermarkExp=`ts` - CASE WHEN true THEN INTERVAL &#39;10&#39; SECOND ELSE INTERVAL &#39;2&#39; SECOND END]">
+    <Resource name="sql">
+      <![CDATA[SELECT a, b, ts FROM t1]]>
+    </Resource>
+    <Resource name="ast">
+      <![CDATA[
+LogicalProject(a=[$0], b=[$1], ts=[$5])
++- LogicalWatermarkAssigner(rowtime=[ts], watermark=[-($5, 10000:INTERVAL SECOND)])
+   +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], t=[$4], ts=[$4])
+      +- LogicalTableScan(table=[[default_catalog, default_database, t1]])
+]]>
+    </Resource>
+    <Resource name="optimized exec plan">
+      <![CDATA[
+TableSourceScan(table=[[default_catalog, default_database, t1, project=[a, b, t], metadata=[], watermark=[-(t, 10000:INTERVAL SECOND)], watermarkEmitStrategy=[on-periodic]]], fields=[a, b, t])
 ]]>
     </Resource>
   </TestCase>
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.scala
index 1fc524d4ae1..913d8f21482 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/SourceWatermarkTest.scala
@@ -21,6 +21,8 @@ import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedScalarFunctio
 import org.apache.flink.table.planner.utils.TableTestBase
 
 import org.junit.jupiter.api.{BeforeEach, Test}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ValueSource
 
 /** Tests for watermark push down. */
 class SourceWatermarkTest extends TableTestBase {
@@ -179,7 +181,37 @@ class SourceWatermarkTest extends TableTestBase {
          |  `d`  INT,
          |  `t`  TIMESTAMP(3),
          |  `ts` AS `t`,
-         |  WATERMARK FOR `ts` AS `ts`  - INTERVAL '10' SECOND
+         |  WATERMARK FOR `ts` AS `ts` - INTERVAL '10' SECOND
+         |) WITH (
+         |  'connector' = 'values',
+         |  'enable-watermark-push-down' = 'true',
+         |  'bounded' = 'false',
+         |  'disable-lookup' = 'true'
+         |)
+       """.stripMargin
+    util.tableEnv.executeSql(sourceDDL)
+    util.verifyExecPlan("SELECT a, b, ts FROM t1")
+  }
+
+  @ParameterizedTest
+  @ValueSource(
+    strings = Array[String](
+      "`ts` - ( - ( - (INTERVAL '10' SECOND)))",
+      "`ts` - ARRAY[INTERVAL '10' SECOND, INTERVAL '1' SECOND][1]",
+      "`ts` - CASE WHEN true THEN INTERVAL '10' SECOND ELSE INTERVAL '2' SECOND END"
+    ))
+  def testProjectTransposeWatermarkAssignerWithSimplifiableWatermarks(
+      watermarkExp: String): Unit = {
+    val sourceDDL =
+      s"""
+         |CREATE TEMPORARY TABLE `t1` (
+         |  `a`  VARCHAR,
+         |  `b`  VARCHAR,
+         |  `c`  VARCHAR,
+         |  `d`  INT,
+         |  `t`  TIMESTAMP(3),
+         |  `ts` AS `t`,
+         |  WATERMARK FOR `ts` AS $watermarkExp
          |) WITH (
          |  'connector' = 'values',
          |  'enable-watermark-push-down' = 'true',