You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2024/10/16 07:02:41 UTC

(flink) branch master updated: [FLINK-36543] Fix string serialization for OVER windows (#25526)

This is an automated email from the ASF dual-hosted git repository.

dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 55cc36a8155 [FLINK-36543] Fix string serialization for OVER windows (#25526)
55cc36a8155 is described below

commit 55cc36a81553523b90b58d89c11ec5ba495f23f4
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Wed Oct 16 09:02:33 2024 +0200

    [FLINK-36543] Fix string serialization for OVER windows (#25526)
---
 .../functions/BuiltInFunctionDefinitions.java      |  1 +
 .../flink/table/functions/CallSyntaxUtils.java     | 57 +++++++++++++++
 .../flink/table/functions/SqlCallSyntax.java       | 19 +++++
 .../table/api/QueryOperationSqlExecutionTest.java  |  4 +-
 .../api/QueryOperationSqlSerializationTest.java    |  4 +-
 .../table/api/QueryOperationTestPrograms.java      | 85 ++++++++++++++++++++++
 6 files changed, 168 insertions(+), 2 deletions(-)

diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
index dadb9ef12bd..a6ab7984457 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
@@ -2569,6 +2569,7 @@ public final class BuiltInFunctionDefinitions {
                     .kind(OTHER)
                     .inputTypeStrategy(SpecificInputTypeStrategies.OVER)
                     .outputTypeStrategy(TypeStrategies.argument(0))
+                    .callSyntax(SqlCallSyntax.OVER)
                     .build();
 
     // --------------------------------------------------------------------------------------------
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/CallSyntaxUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/CallSyntaxUtils.java
index d285db68be0..82c052d12a9 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/CallSyntaxUtils.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/CallSyntaxUtils.java
@@ -19,10 +19,16 @@
 package org.apache.flink.table.functions;
 
 import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.OverWindowRange;
 import org.apache.flink.table.expressions.FieldReferenceExpression;
 import org.apache.flink.table.expressions.ResolvedExpression;
 import org.apache.flink.table.expressions.TableSymbol;
 import org.apache.flink.table.expressions.ValueLiteralExpression;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeFamily;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+
+import java.util.Optional;
 
 /** Utility functions that can be used for writing {@link SqlCallSyntax}. */
 @Internal
@@ -45,5 +51,56 @@ class CallSyntaxUtils {
         return ((ValueLiteralExpression) operands).getValueAs(clazz).get();
     }
 
+    static String overRangeToSerializableString(
+            ResolvedExpression preceding, ResolvedExpression following) {
+        return String.format(
+                "%s BETWEEN %s AND %s",
+                isRowsRange(preceding) ? "ROWS" : "RANGE",
+                toStringPrecedingOrFollowing(preceding, true),
+                toStringPrecedingOrFollowing(following, false));
+    }
+
+    private static String toStringPrecedingOrFollowing(
+            ResolvedExpression precedingOrFollowing, boolean isPreceding) {
+        final String suffix = isPreceding ? "PRECEDING" : "FOLLOWING";
+        return Optional.of(precedingOrFollowing)
+                .flatMap(
+                        expr -> {
+                            if (expr instanceof ValueLiteralExpression) {
+                                return ((ValueLiteralExpression) expr)
+                                        .getValueAs(OverWindowRange.class)
+                                        .map(
+                                                r -> {
+                                                    switch (r) {
+                                                        case CURRENT_ROW:
+                                                        case CURRENT_RANGE:
+                                                            return "CURRENT ROW";
+                                                        case UNBOUNDED_ROW:
+                                                        case UNBOUNDED_RANGE:
+                                                            return "UNBOUNDED " + suffix;
+                                                        default:
+                                                            throw new IllegalStateException(
+                                                                    "Unknown window range: " + r);
+                                                    }
+                                                });
+                            } else {
+                                return Optional.empty();
+                            }
+                        })
+                .orElseGet(() -> precedingOrFollowing.asSerializableString() + " " + suffix);
+    }
+
+    private static boolean isRowsRange(ResolvedExpression expression) {
+        LogicalType logicalType = expression.getOutputDataType().getLogicalType();
+        boolean isSymbol = logicalType.is(LogicalTypeRoot.SYMBOL);
+        if (isSymbol) {
+            OverWindowRange windowRange = getSymbolLiteral(expression, OverWindowRange.class);
+            return windowRange == OverWindowRange.CURRENT_ROW
+                    || windowRange == OverWindowRange.UNBOUNDED_ROW;
+        }
+
+        return logicalType.is(LogicalTypeFamily.NUMERIC);
+    }
+
     private CallSyntaxUtils() {}
 }
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/SqlCallSyntax.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/SqlCallSyntax.java
index ba2a12406cc..96e47d6e8ad 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/SqlCallSyntax.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/SqlCallSyntax.java
@@ -316,4 +316,23 @@ public interface SqlCallSyntax {
                             CallSyntaxUtils.asSerializableOperand(operands.get(2)));
                 }
             };
+
+    SqlCallSyntax OVER =
+            ((sqlName, operands) -> {
+                String projection = operands.get(0).asSerializableString();
+                String order = operands.get(1).asSerializableString();
+                String rangeBounds =
+                        CallSyntaxUtils.overRangeToSerializableString(
+                                operands.get(2), operands.get(3));
+                if (operands.size() == 4) {
+                    return String.format("%s OVER(ORDER BY %s %s)", projection, order, rangeBounds);
+                } else {
+                    return String.format(
+                            "%s OVER(PARTITION BY %s ORDER BY %s %s)",
+                            projection,
+                            CallSyntaxUtils.asSerializableOperand(operands.get(4)),
+                            order,
+                            rangeBounds);
+                }
+            });
 }
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlExecutionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlExecutionTest.java
index 300120eb644..447cd6dcde8 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlExecutionTest.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlExecutionTest.java
@@ -71,7 +71,9 @@ public class QueryOperationSqlExecutionTest implements TableTestProgramRunner {
                 QueryOperationTestPrograms.SORT_LIMIT_DESC,
                 QueryOperationTestPrograms.GROUP_BY_UDF_WITH_MERGE,
                 QueryOperationTestPrograms.NON_WINDOW_INNER_JOIN,
-                QueryOperationTestPrograms.SQL_QUERY_OPERATION);
+                QueryOperationTestPrograms.SQL_QUERY_OPERATION,
+                QueryOperationTestPrograms.OVER_WINDOW_RANGE,
+                QueryOperationTestPrograms.OVER_WINDOW_ROWS);
     }
 
     @ParameterizedTest
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlSerializationTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlSerializationTest.java
index 6acaf2c7a1d..8ed52566bb3 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlSerializationTest.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlSerializationTest.java
@@ -60,7 +60,9 @@ public class QueryOperationSqlSerializationTest implements TableTestProgramRunne
                 QueryOperationTestPrograms.WINDOW_AGGREGATE_QUERY_OPERATION,
                 QueryOperationTestPrograms.UNION_ALL_QUERY_OPERATION,
                 QueryOperationTestPrograms.LATERAL_JOIN_QUERY_OPERATION,
-                QueryOperationTestPrograms.SQL_QUERY_OPERATION);
+                QueryOperationTestPrograms.SQL_QUERY_OPERATION,
+                QueryOperationTestPrograms.OVER_WINDOW_RANGE,
+                QueryOperationTestPrograms.OVER_WINDOW_ROWS);
     }
 
     @ParameterizedTest
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationTestPrograms.java
index b4523da6980..f55bdfe6259 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationTestPrograms.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationTestPrograms.java
@@ -740,4 +740,89 @@ public class QueryOperationTestPrograms {
                             },
                             "MySink")
                     .build();
+
+    static final TableTestProgram OVER_WINDOW_RANGE =
+            TableTestProgram.of("over-window-range", "test over window with time range")
+                    .setupTableSource(
+                            SourceTestStep.newBuilder("data")
+                                    .addSchema(
+                                            "k string",
+                                            "v bigint",
+                                            "ts TIMESTAMP_LTZ(3)",
+                                            "WATERMARK for `ts` AS `ts`")
+                                    .producedBeforeRestore(
+                                            Row.of("Apple", 5L, dayOfSeconds(0)),
+                                            Row.of("Apple", 4L, dayOfSeconds(1)))
+                                    .producedAfterRestore(Row.of("Apple", 3L, dayOfSeconds(2)))
+                                    .build())
+                    .setupTableSink(
+                            SinkTestStep.newBuilder("sink")
+                                    .addSchema("k string", "v bigint", "ts TIMESTAMP_LTZ(3)")
+                                    .consumedBeforeRestore(
+                                            Row.of("Apple", 5L, dayOfSeconds(0)),
+                                            Row.of("Apple", 4L, dayOfSeconds(1)))
+                                    .consumedAfterRestore(Row.of("Apple", 3L, dayOfSeconds(2)))
+                                    .build())
+                    .runSql(
+                            "SELECT `k`, (LAST_VALUE(`v`) OVER(PARTITION BY `k` ORDER BY `ts` RANGE BETWEEN INTERVAL '0 00:00:02.0' DAY TO SECOND(3) PRECEDING AND CURRENT ROW)) AS `_c1`, `ts` FROM (\n"
+                                    + "    SELECT `k`, `v`, `ts` FROM `default_catalog`.`default_database`.`data`\n"
+                                    + ")")
+                    .runTableApi(
+                            tableEnvAccessor ->
+                                    tableEnvAccessor
+                                            .from("data")
+                                            .window(
+                                                    Over.partitionBy($("k"))
+                                                            .orderBy($("ts"))
+                                                            .preceding(lit(2).second())
+                                                            .as("w"))
+                                            .select(
+                                                    $("k"),
+                                                    $("v").lastValue().over($("w")),
+                                                    $("ts")),
+                            "sink")
+                    .build();
+
+    static final TableTestProgram OVER_WINDOW_ROWS =
+            TableTestProgram.of("over-window-rows", "test over window with rows range")
+                    .setupTableSource(
+                            SourceTestStep.newBuilder("data")
+                                    .addSchema(
+                                            "k string",
+                                            "v bigint",
+                                            "ts TIMESTAMP_LTZ(3)",
+                                            "WATERMARK for `ts` AS `ts`")
+                                    .producedBeforeRestore(
+                                            Row.of("Apple", 5L, dayOfSeconds(0)),
+                                            Row.of("Apple", 4L, dayOfSeconds(1)))
+                                    .producedAfterRestore(Row.of("Apple", 3L, dayOfSeconds(2)))
+                                    .build())
+                    .setupTableSink(
+                            SinkTestStep.newBuilder("sink")
+                                    .addSchema("k string", "v bigint", "ts TIMESTAMP_LTZ(3)")
+                                    .consumedBeforeRestore(
+                                            Row.of("Apple", 5L, dayOfSeconds(0)),
+                                            Row.of("Apple", 4L, dayOfSeconds(1)))
+                                    .consumedAfterRestore(Row.of("Apple", 3L, dayOfSeconds(2)))
+                                    .build())
+                    .runSql(
+                            "SELECT `k`, (LAST_VALUE(`v`) OVER(PARTITION BY `k` ORDER BY `ts` "
+                                    + "ROWS BETWEEN CAST(2 AS BIGINT) PRECEDING AND CURRENT ROW)) AS `_c1`, `ts` FROM (\n"
+                                    + "    SELECT `k`, `v`, `ts` FROM `default_catalog`.`default_database`.`data`\n"
+                                    + ")")
+                    .runTableApi(
+                            tableEnvAccessor ->
+                                    tableEnvAccessor
+                                            .from("data")
+                                            .window(
+                                                    Over.partitionBy($("k"))
+                                                            .orderBy($("ts"))
+                                                            .preceding(lit(2L))
+                                                            .as("w"))
+                                            .select(
+                                                    $("k"),
+                                                    $("v").lastValue().over($("w")),
+                                                    $("ts")),
+                            "sink")
+                    .build();
 }