You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2024/10/16 07:02:41 UTC
(flink) branch master updated: [FLINK-36543] Fix string serialization for OVER windows (#25526)
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 55cc36a8155 [FLINK-36543] Fix string serialization for OVER windows (#25526)
55cc36a8155 is described below
commit 55cc36a81553523b90b58d89c11ec5ba495f23f4
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Wed Oct 16 09:02:33 2024 +0200
[FLINK-36543] Fix string serialization for OVER windows (#25526)
---
.../functions/BuiltInFunctionDefinitions.java | 1 +
.../flink/table/functions/CallSyntaxUtils.java | 57 +++++++++++++++
.../flink/table/functions/SqlCallSyntax.java | 19 +++++
.../table/api/QueryOperationSqlExecutionTest.java | 4 +-
.../api/QueryOperationSqlSerializationTest.java | 4 +-
.../table/api/QueryOperationTestPrograms.java | 85 ++++++++++++++++++++++
6 files changed, 168 insertions(+), 2 deletions(-)
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
index dadb9ef12bd..a6ab7984457 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
@@ -2569,6 +2569,7 @@ public final class BuiltInFunctionDefinitions {
.kind(OTHER)
.inputTypeStrategy(SpecificInputTypeStrategies.OVER)
.outputTypeStrategy(TypeStrategies.argument(0))
+ .callSyntax(SqlCallSyntax.OVER)
.build();
// --------------------------------------------------------------------------------------------
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/CallSyntaxUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/CallSyntaxUtils.java
index d285db68be0..82c052d12a9 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/CallSyntaxUtils.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/CallSyntaxUtils.java
@@ -19,10 +19,16 @@
package org.apache.flink.table.functions;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.OverWindowRange;
import org.apache.flink.table.expressions.FieldReferenceExpression;
import org.apache.flink.table.expressions.ResolvedExpression;
import org.apache.flink.table.expressions.TableSymbol;
import org.apache.flink.table.expressions.ValueLiteralExpression;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeFamily;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+
+import java.util.Optional;
/** Utility functions that can be used for writing {@link SqlCallSyntax}. */
@Internal
@@ -45,5 +51,56 @@ class CallSyntaxUtils {
return ((ValueLiteralExpression) operands).getValueAs(clazz).get();
}
+ static String overRangeToSerializableString(
+ ResolvedExpression preceding, ResolvedExpression following) {
+ return String.format(
+ "%s BETWEEN %s AND %s",
+ isRowsRange(preceding) ? "ROWS" : "RANGE",
+ toStringPrecedingOrFollowing(preceding, true),
+ toStringPrecedingOrFollowing(following, false));
+ }
+
+ private static String toStringPrecedingOrFollowing(
+ ResolvedExpression precedingOrFollowing, boolean isPreceding) {
+ final String suffix = isPreceding ? "PRECEDING" : "FOLLOWING";
+ return Optional.of(precedingOrFollowing)
+ .flatMap(
+ expr -> {
+ if (expr instanceof ValueLiteralExpression) {
+ return ((ValueLiteralExpression) expr)
+ .getValueAs(OverWindowRange.class)
+ .map(
+ r -> {
+ switch (r) {
+ case CURRENT_ROW:
+ case CURRENT_RANGE:
+ return "CURRENT ROW";
+ case UNBOUNDED_ROW:
+ case UNBOUNDED_RANGE:
+ return "UNBOUNDED " + suffix;
+ default:
+ throw new IllegalStateException(
+ "Unknown window range: " + r);
+ }
+ });
+ } else {
+ return Optional.empty();
+ }
+ })
+ .orElseGet(() -> precedingOrFollowing.asSerializableString() + " " + suffix);
+ }
+
+ private static boolean isRowsRange(ResolvedExpression expression) {
+ LogicalType logicalType = expression.getOutputDataType().getLogicalType();
+ boolean isSymbol = logicalType.is(LogicalTypeRoot.SYMBOL);
+ if (isSymbol) {
+ OverWindowRange windowRange = getSymbolLiteral(expression, OverWindowRange.class);
+ return windowRange == OverWindowRange.CURRENT_ROW
+ || windowRange == OverWindowRange.UNBOUNDED_ROW;
+ }
+
+ return logicalType.is(LogicalTypeFamily.NUMERIC);
+ }
+
private CallSyntaxUtils() {}
}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/SqlCallSyntax.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/SqlCallSyntax.java
index ba2a12406cc..96e47d6e8ad 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/SqlCallSyntax.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/SqlCallSyntax.java
@@ -316,4 +316,23 @@ public interface SqlCallSyntax {
CallSyntaxUtils.asSerializableOperand(operands.get(2)));
}
};
+
+ SqlCallSyntax OVER =
+ ((sqlName, operands) -> {
+ String projection = operands.get(0).asSerializableString();
+ String order = operands.get(1).asSerializableString();
+ String rangeBounds =
+ CallSyntaxUtils.overRangeToSerializableString(
+ operands.get(2), operands.get(3));
+ if (operands.size() == 4) {
+ return String.format("%s OVER(ORDER BY %s %s)", projection, order, rangeBounds);
+ } else {
+ return String.format(
+ "%s OVER(PARTITION BY %s ORDER BY %s %s)",
+ projection,
+ CallSyntaxUtils.asSerializableOperand(operands.get(4)),
+ order,
+ rangeBounds);
+ }
+ });
}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlExecutionTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlExecutionTest.java
index 300120eb644..447cd6dcde8 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlExecutionTest.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlExecutionTest.java
@@ -71,7 +71,9 @@ public class QueryOperationSqlExecutionTest implements TableTestProgramRunner {
QueryOperationTestPrograms.SORT_LIMIT_DESC,
QueryOperationTestPrograms.GROUP_BY_UDF_WITH_MERGE,
QueryOperationTestPrograms.NON_WINDOW_INNER_JOIN,
- QueryOperationTestPrograms.SQL_QUERY_OPERATION);
+ QueryOperationTestPrograms.SQL_QUERY_OPERATION,
+ QueryOperationTestPrograms.OVER_WINDOW_RANGE,
+ QueryOperationTestPrograms.OVER_WINDOW_ROWS);
}
@ParameterizedTest
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlSerializationTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlSerializationTest.java
index 6acaf2c7a1d..8ed52566bb3 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlSerializationTest.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationSqlSerializationTest.java
@@ -60,7 +60,9 @@ public class QueryOperationSqlSerializationTest implements TableTestProgramRunne
QueryOperationTestPrograms.WINDOW_AGGREGATE_QUERY_OPERATION,
QueryOperationTestPrograms.UNION_ALL_QUERY_OPERATION,
QueryOperationTestPrograms.LATERAL_JOIN_QUERY_OPERATION,
- QueryOperationTestPrograms.SQL_QUERY_OPERATION);
+ QueryOperationTestPrograms.SQL_QUERY_OPERATION,
+ QueryOperationTestPrograms.OVER_WINDOW_RANGE,
+ QueryOperationTestPrograms.OVER_WINDOW_ROWS);
}
@ParameterizedTest
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationTestPrograms.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationTestPrograms.java
index b4523da6980..f55bdfe6259 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationTestPrograms.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/QueryOperationTestPrograms.java
@@ -740,4 +740,89 @@ public class QueryOperationTestPrograms {
},
"MySink")
.build();
+
+ static final TableTestProgram OVER_WINDOW_RANGE =
+ TableTestProgram.of("over-window-range", "test over window with time range")
+ .setupTableSource(
+ SourceTestStep.newBuilder("data")
+ .addSchema(
+ "k string",
+ "v bigint",
+ "ts TIMESTAMP_LTZ(3)",
+ "WATERMARK for `ts` AS `ts`")
+ .producedBeforeRestore(
+ Row.of("Apple", 5L, dayOfSeconds(0)),
+ Row.of("Apple", 4L, dayOfSeconds(1)))
+ .producedAfterRestore(Row.of("Apple", 3L, dayOfSeconds(2)))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("k string", "v bigint", "ts TIMESTAMP_LTZ(3)")
+ .consumedBeforeRestore(
+ Row.of("Apple", 5L, dayOfSeconds(0)),
+ Row.of("Apple", 4L, dayOfSeconds(1)))
+ .consumedAfterRestore(Row.of("Apple", 3L, dayOfSeconds(2)))
+ .build())
+ .runSql(
+ "SELECT `k`, (LAST_VALUE(`v`) OVER(PARTITION BY `k` ORDER BY `ts` RANGE BETWEEN INTERVAL '0 00:00:02.0' DAY TO SECOND(3) PRECEDING AND CURRENT ROW)) AS `_c1`, `ts` FROM (\n"
+ + " SELECT `k`, `v`, `ts` FROM `default_catalog`.`default_database`.`data`\n"
+ + ")")
+ .runTableApi(
+ tableEnvAccessor ->
+ tableEnvAccessor
+ .from("data")
+ .window(
+ Over.partitionBy($("k"))
+ .orderBy($("ts"))
+ .preceding(lit(2).second())
+ .as("w"))
+ .select(
+ $("k"),
+ $("v").lastValue().over($("w")),
+ $("ts")),
+ "sink")
+ .build();
+
+ static final TableTestProgram OVER_WINDOW_ROWS =
+ TableTestProgram.of("over-window-rows", "test over window with rows range")
+ .setupTableSource(
+ SourceTestStep.newBuilder("data")
+ .addSchema(
+ "k string",
+ "v bigint",
+ "ts TIMESTAMP_LTZ(3)",
+ "WATERMARK for `ts` AS `ts`")
+ .producedBeforeRestore(
+ Row.of("Apple", 5L, dayOfSeconds(0)),
+ Row.of("Apple", 4L, dayOfSeconds(1)))
+ .producedAfterRestore(Row.of("Apple", 3L, dayOfSeconds(2)))
+ .build())
+ .setupTableSink(
+ SinkTestStep.newBuilder("sink")
+ .addSchema("k string", "v bigint", "ts TIMESTAMP_LTZ(3)")
+ .consumedBeforeRestore(
+ Row.of("Apple", 5L, dayOfSeconds(0)),
+ Row.of("Apple", 4L, dayOfSeconds(1)))
+ .consumedAfterRestore(Row.of("Apple", 3L, dayOfSeconds(2)))
+ .build())
+ .runSql(
+ "SELECT `k`, (LAST_VALUE(`v`) OVER(PARTITION BY `k` ORDER BY `ts` "
+ + "ROWS BETWEEN CAST(2 AS BIGINT) PRECEDING AND CURRENT ROW)) AS `_c1`, `ts` FROM (\n"
+ + " SELECT `k`, `v`, `ts` FROM `default_catalog`.`default_database`.`data`\n"
+ + ")")
+ .runTableApi(
+ tableEnvAccessor ->
+ tableEnvAccessor
+ .from("data")
+ .window(
+ Over.partitionBy($("k"))
+ .orderBy($("ts"))
+ .preceding(lit(2L))
+ .as("w"))
+ .select(
+ $("k"),
+ $("v").lastValue().over($("w")),
+ $("ts")),
+ "sink")
+ .build();
}