You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by jc...@apache.org on 2024/01/25 02:05:59 UTC
(flink) branch master updated: [FLINK-24024][table-planner] Support session window TVF
This is an automated email from the ASF dual-hosted git repository.
jchan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 4c1ae285fb6 [FLINK-24024][table-planner] Support session window TVF
4c1ae285fb6 is described below
commit 4c1ae285fb672bbe65f894251ee9a7cfc49db6a6
Author: Xuyang <xy...@163.com>
AuthorDate: Thu Jan 25 10:05:51 2024 +0800
[FLINK-24024][table-planner] Support session window TVF
This closes #23505
---
.../apache/calcite/sql2rel/SqlToRelConverter.java | 89 ++--
.../planner/calcite/FlinkConvertletTable.java | 114 +++++
.../planner/calcite/RexSetSemanticsTableCall.java | 95 ++++
.../functions/sql/FlinkSqlOperatorTable.java | 1 +
.../functions/sql/SqlSessionTableFunction.java | 103 +++++
.../planner/plan/logical/CumulativeWindowSpec.java | 2 +-
.../planner/plan/logical/HoppingWindowSpec.java | 2 +-
.../planner/plan/logical/SessionWindowSpec.java | 100 ++++
.../logical/SliceAttachedWindowingStrategy.java | 2 +-
.../logical/TimeAttributeWindowingStrategy.java | 2 +-
.../planner/plan/logical/TumblingWindowSpec.java | 2 +-
.../logical/WindowAttachedWindowingStrategy.java | 2 +-
.../table/planner/plan/logical/WindowSpec.java | 5 +-
.../exec/common/CommonExecWindowTableFunction.java | 7 +
.../exec/stream/StreamExecWindowAggregate.java | 58 +++
.../ProjectWindowTableFunctionTransposeRule.java | 20 +-
.../TwoStageOptimizedWindowAggregateRule.java | 7 +
.../planner/plan/trait/RelWindowProperties.java | 13 +
.../plan/metadata/FlinkRelMdWindowProperties.scala | 50 +-
.../common/CommonPhysicalWindowTableFunction.scala | 4 +-
.../StreamPhysicalLocalWindowAggregate.scala | 14 +-
.../plan/rules/logical/SplitAggregateRule.scala | 8 +-
.../BatchPhysicalWindowTableFunctionRule.scala | 2 +-
...indowTableFunctionIntoWindowAggregateRule.scala | 58 ++-
.../StreamPhysicalWindowTableFunctionRule.scala | 34 +-
.../table/planner/plan/utils/WindowUtil.scala | 79 +++-
.../exec/stream/WindowAggregateJsonPlanTest.java | 76 ++++
.../stream/jsonplan/WindowAggregateJsonITCase.java | 21 +
.../testEventTimeSessionWindowWithPartitionKey.out | 373 +++++++++++++++
...stEventTimeSessionWindowWithoutPartitionKey.out | 355 +++++++++++++++
.../testProcTimeSessionWindowWithPartitionKey.out | 460 +++++++++++++++++++
...estProcTimeSessionWindowWithoutPartitionKey.out | 432 ++++++++++++++++++
.../plan/stream/sql/agg/WindowAggregateTest.xml | 504 +++++++++++++++++++++
.../plan/stream/sql/agg/WindowAggregateTest.scala | 226 ++++++++-
.../runtime/stream/sql/WindowAggregateITCase.scala | 108 +++++
35 files changed, 3368 insertions(+), 60 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index 5e9010a15f5..e36b5e91f49 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -46,8 +46,6 @@ import org.apache.calcite.prepare.RelOptTableImpl;
import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollationTraitDef;
import org.apache.calcite.rel.RelCollations;
-import org.apache.calcite.rel.RelDistribution;
-import org.apache.calcite.rel.RelDistributions;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelRoot;
@@ -236,12 +234,13 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* <p>FLINK modifications are at lines
*
* <ol>
- * <li>Added in FLINK-29081, FLINK-28682, FLINK-33395: Lines 654 ~ 671
- * <li>Added in FLINK-28682: Lines 2294 ~ 2311
- * <li>Added in FLINK-28682: Lines 2348 ~ 2376
- * <li>Added in FLINK-20873: Lines 5489 ~ 5498
- * <li>Added in FLINK-32474: Lines 2846 ~ 2858
- * <li>Added in FLINK-32474: Lines 2958 ~ 2992
+ * <li>Added in FLINK-29081, FLINK-28682, FLINK-33395: Lines 653 ~ 670
+ * <li>Added in Flink-24024: Lines 1434 ~ 1444, Lines 1458 ~ 1500
+ * <li>Added in FLINK-28682: Lines 2322 ~ 2339
+ * <li>Added in FLINK-28682: Lines 2376 ~ 2404
+ * <li>Added in FLINK-32474: Lines 2874 ~ 2886
+ * <li>Added in FLINK-32474: Lines 2986 ~ 3020
+ * <li>Added in FLINK-20873: Lines 5518 ~ 5527
* </ol>
*/
@SuppressWarnings("UnstableApiUsage")
@@ -1432,9 +1431,17 @@ public class SqlToRelConverter {
bb.cursors.add(converted.r);
return;
case SET_SEMANTICS_TABLE:
- if (!config.isExpand()) {
- return;
- }
+ // ----- FLINK MODIFICATION BEGIN -----
+ // We always expand the SET SEMANTICS TABLE for two reasons:
+ // 1. Calcite has a bug when not expanding the SET SEMANTICS TABLE. For more
+ // information, see CALCITE-6204.
+ // 2. Currently, Flink’s built-in Session Window TVF is the only PTF with SET
+ // SEMANTICS. We will expand it by default, like other built-in window TVFs, to
+ // reuse some subsequent processing and optimization logic.
+ // if (!config.isExpand()) {
+ // return;
+ // }
+ // ----- FLINK MODIFICATION END -----
substituteSubQueryOfSetSemanticsInputTable(bb, subQuery);
return;
default:
@@ -1447,30 +1454,52 @@ public class SqlToRelConverter {
SqlNode query;
call = (SqlBasicCall) subQuery.node;
query = call.operand(0);
- final SqlValidatorScope innerTableScope =
- (query instanceof SqlSelect) ? validator().getSelectScope((SqlSelect) query) : null;
- final Blackboard setSemanticsTableBb = createBlackboard(innerTableScope, null, false);
+
+ // FLINK MODIFICATION BEGIN
+
+ // We modified it for two reasons:
+ // 1. In Flink, Exchange nodes should not appear in the logical stage, which will bring
+ // uncertainty to the implementation of plan optimization in the current logical stage.
+ // Instead, Flink will add exchanges based on traits during the physical phase.
+ // 2. Currently, Flink’s built-in Session Window TVF is the only SET SEMANTICS
+ // TABLE. We will convert it into the same plan tree as other Window TVFs. The partition key
+ // and order key will be recorded using a custom RexCall when subsequently converting the
+ // SqlCall of SET SEMANTICS TABLE. See more at
+ // FlinkConvertletTable#convertSetSemanticsWindowTableFunction
+
final RelNode inputOfSetSemanticsTable =
convertQueryRecursive(query, false, null).project();
- requireNonNull(inputOfSetSemanticsTable, () -> "input RelNode is null for query " + query);
- SqlNodeList partitionList = call.operand(1);
- final ImmutableBitSet partitionKeys =
- buildPartitionKeys(setSemanticsTableBb, partitionList);
+ relBuilder.push(inputOfSetSemanticsTable);
+
+ // final SqlValidatorScope innerTableScope =
+ // (query instanceof SqlSelect) ? validator().getSelectScope((SqlSelect) query) :
+ // null;
+ // final Blackboard setSemanticsTableBb = createBlackboard(innerTableScope, null, false);
+ // final RelNode inputOfSetSemanticsTable =
+ // convertQueryRecursive(query, false, null).project();
+ // relBuilder.push(inputOfSetSemanticsTable);
+ // requireNonNull(inputOfSetSemanticsTable, () -> "input RelNode is null for query " +
+ // query);
+ // SqlNodeList partitionList = call.operand(1);
+ // final ImmutableBitSet partitionKeys =
+ // buildPartitionKeys(setSemanticsTableBb, partitionList);
// For set semantics table, distribution is singleton if does not specify
// partition keys
- RelDistribution distribution =
- partitionKeys.isEmpty()
- ? RelDistributions.SINGLETON
- : RelDistributions.hash(partitionKeys.asList());
+ // RelDistribution distribution =
+ // partitionKeys.isEmpty()
+ // ? RelDistributions.SINGLETON
+ // : RelDistributions.hash(partitionKeys.asList());
// ORDER BY
- final SqlNodeList orderList = call.operand(2);
- final RelCollation orders = buildCollation(setSemanticsTableBb, orderList);
- relBuilder.push(inputOfSetSemanticsTable);
- if (orderList.isEmpty()) {
- relBuilder.exchange(distribution);
- } else {
- relBuilder.sortExchange(distribution, orders);
- }
+ // final SqlNodeList orderList = call.operand(2);
+ // final RelCollation orders = buildCollation(setSemanticsTableBb, orderList);
+ // if (orderList.isEmpty()) {
+ // relBuilder.exchange(distribution);
+ // } else {
+ // relBuilder.sortExchange(distribution, orders);
+ // }
+
+ // FLINK MODIFICATION END
+
RelNode tableRel = relBuilder.build();
subQuery.expr = bb.register(tableRel, JoinRelType.LEFT);
// This is used when converting window table functions:
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkConvertletTable.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkConvertletTable.java
index d92555fa545..8f319dd1e23 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkConvertletTable.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkConvertletTable.java
@@ -19,16 +19,24 @@
package org.apache.flink.table.planner.calcite;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.api.TableException;
import org.apache.flink.table.planner.functions.sql.FlinkSqlOperatorTable;
+import org.apache.flink.table.planner.functions.sql.SqlSessionTableFunction;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlDataTypeSpec;
import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.SqlWindowTableFunction;
import org.apache.calcite.sql.validate.SqlValidatorImpl;
import org.apache.calcite.sql2rel.SqlRexContext;
import org.apache.calcite.sql2rel.SqlRexConvertlet;
@@ -36,6 +44,9 @@ import org.apache.calcite.sql2rel.SqlRexConvertletTable;
import org.apache.calcite.sql2rel.StandardConvertletTable;
import java.util.Collections;
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
/**
* Custom Flink {@link SqlRexConvertletTable} to add custom {@link SqlNode} to {@link RexNode}
@@ -53,6 +64,11 @@ public class FlinkConvertletTable implements SqlRexConvertletTable {
if (call.getOperator().isName("TRY_CAST", false)) {
return this::convertTryCast;
}
+
+ if (isSetSemanticsWindowTableFunction(call)) {
+ return this::convertSetSemanticsWindowTableFunction;
+ }
+
return StandardConvertletTable.INSTANCE.get(call);
}
@@ -88,4 +104,102 @@ public class FlinkConvertletTable implements SqlRexConvertletTable {
.makeCall(
type, FlinkSqlOperatorTable.TRY_CAST, Collections.singletonList(valueRex));
}
+
+ private boolean isSetSemanticsWindowTableFunction(SqlCall call) {
+ if (!(call.getOperator() instanceof SqlWindowTableFunction)) {
+ return false;
+ }
+ List<SqlNode> operands = call.getOperandList();
+ return !operands.isEmpty() && operands.get(0).getKind() == SqlKind.SET_SEMANTICS_TABLE;
+ }
+
+ /**
+ * Due to CALCITE-6204, we need to manually extract partition keys and order keys and convert
+ * them to {@link RexSetSemanticsTableCall}.
+ *
+ * <p>Take `SESSION(TABLE my_table PARTITION BY (b, a), DESCRIPTOR(rowtime), INTERVAL '10'
+ * MINUTE)` as an example.
+ *
+ * <p>The original SqlNode tree after syntax parse looks like
+ *
+ * <pre>
+ * SqlBasicCall: SESSION
+ * ├─ SqlBasicCall: SET_SEMANTICS_TABLE
+ * │ ├─ SqlSelect: "SELECT ... FROM ..."
+ * │ ├─ SqlNodeList: (PARTITION KEY)
+ * │ │ ├─ SqlIdentifier: "b"
+ * │ │ └─ SqlIdentifier: "a"
+ * │ └─ SqlNodeList: (ORDER KEY)
+ * ├─ SqlBasicCall: DESCRIPTOR(`rowtime`)
+ * │ └─ SqlIdentifier: "rowtime"
+ * └─ SqlInternalLiteral: INTERVAL '5' MINUTE
+ * </pre>
+ *
+ * <p>Calcite will skip the first operand of SESSION operator, which leads to the following
+ * wrong rex call
+ *
+ * <pre>
+ * RexCall: SESSION
+ * ├─ RexCall: DESCRIPTOR(`rowtime`)
+ * │ └─ RexInputRef: `rowtime`
+ * └─ RexLiteral: 300000:INTERVAL MINUTE
+ * </pre>
+ *
+ * <p>As a workaround, we flatten the inner sql call and convert it to a customized {@link
+ * RexSetSemanticsTableCall} to preserve partition keys and order keys
+ *
+ * <pre>
+ * RexSetSemanticsTableCall: SESSION
+ * ├─ PartitionKeys: [1, 0]
+ * ├─ OrderKeys: []
+ * ├─ RexCall: DESCRIPTOR(`rowtime`)
+ * │ └─ RexInputRef: `rowtime`
+ * └─ RexLiteral: 300000:INTERVAL MINUTE
+ * </pre>
+ */
+ private RexNode convertSetSemanticsWindowTableFunction(SqlRexContext cx, final SqlCall call) {
+ checkArgument(
+ call.getOperator() instanceof SqlSessionTableFunction,
+ "Currently, only the SESSION table function is supported in Set Semantics PTF.");
+ SqlSessionTableFunction fun = (SqlSessionTableFunction) call.getOperator();
+
+ List<SqlNode> operands = call.getOperandList();
+
+ SqlBasicCall setSemanticsPTFCall = (SqlBasicCall) operands.get(0);
+ SqlNodeList partitionKeys = setSemanticsPTFCall.operand(1);
+ SqlNodeList orderKeys = setSemanticsPTFCall.operand(2);
+ checkArgument(orderKeys.isEmpty(), "SESSION table function does not support order keys.");
+ RexCall resolvedCall =
+ (RexCall) StandardConvertletTable.INSTANCE.convertWindowFunction(cx, fun, call);
+ int[] partitionKeyRefs = getPartitionKeyIndices(cx, partitionKeys);
+
+ // attach the partition keys and order keys on the custom rex call
+ resolvedCall =
+ new RexSetSemanticsTableCall(
+ resolvedCall.getType(),
+ resolvedCall.getOperator(),
+ resolvedCall.getOperands(),
+ partitionKeyRefs,
+ new int[] {});
+ return resolvedCall;
+ }
+
+ private int[] getPartitionKeyIndices(SqlRexContext cx, SqlNodeList partitions) {
+ final int[] result = new int[partitions.size()];
+
+ for (int i = 0; i < partitions.getList().size(); i++) {
+ RexNode expr = cx.convertExpression(partitions.get(i));
+ result[i] = parseFieldIdx(expr);
+ }
+ return result;
+ }
+
+ private static int parseFieldIdx(RexNode e) {
+ if (SqlKind.INPUT_REF == e.getKind()) {
+ final RexInputRef ref = (RexInputRef) e;
+ return ref.getIndex();
+ }
+ // should not happen
+ throw new TableException("Unsupported partition key with type: " + e.getKind());
+ }
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RexSetSemanticsTableCall.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RexSetSemanticsTableCall.java
new file mode 100644
index 00000000000..f2d2ce07b1e
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RexSetSemanticsTableCall.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.calcite;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlSyntax;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * A special {@link RexCall} that is used to represent a table function with set semantics. See more
+ * details in {@link FlinkConvertletTable#convertSetSemanticsWindowTableFunction}.
+ */
+public class RexSetSemanticsTableCall extends RexCall {
+
+ private final int[] partitionKeys;
+
+ private final int[] orderKeys;
+
+ public RexSetSemanticsTableCall(
+ RelDataType type,
+ SqlOperator operator,
+ List<? extends RexNode> operands,
+ int[] partitionKeys,
+ int[] orderKeys) {
+ super(type, operator, operands);
+ this.partitionKeys = partitionKeys;
+ this.orderKeys = orderKeys;
+ }
+
+ public int[] getPartitionKeys() {
+ return partitionKeys;
+ }
+
+ public int[] getOrderKeys() {
+ return orderKeys;
+ }
+
+ @Override
+ protected String computeDigest(boolean withType) {
+ if ((operands.isEmpty()) && (op.getSyntax() == SqlSyntax.FUNCTION_ID)) {
+ return super.computeDigest(withType);
+ }
+ final StringBuilder sb = new StringBuilder(op.getName());
+ sb.append("(");
+ appendKeys(partitionKeys, "PARTITION BY", sb);
+ appendKeys(orderKeys, "ORDER BY", sb);
+ appendOperands(sb);
+ sb.append(")");
+ if (withType) {
+ sb.append(":");
+
+ // NOTE jvs 16-Jan-2005: for digests, it is very important
+ // to use the full type string.
+ sb.append(type.getFullTypeString());
+ }
+ return sb.toString();
+ }
+
+ private void appendKeys(int[] keys, String prefix, StringBuilder sb) {
+ if (keys.length == 0) {
+ return;
+ }
+ sb.append(
+ Arrays.stream(keys)
+ .mapToObj(key -> "$" + key)
+ .collect(Collectors.joining(", ", prefix + "(", "), ")));
+ }
+
+ public RexSetSemanticsTableCall copy(
+ List<? extends RexNode> newOperands, int[] newPartitionKeys, int[] newOrderKeys) {
+ return new RexSetSemanticsTableCall(type, op, newOperands, newPartitionKeys, newOrderKeys);
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
index 1a98081dd79..c2b3653aa69 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
@@ -1276,6 +1276,7 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
public static final SqlFunction TUMBLE = new SqlTumbleTableFunction();
public static final SqlFunction HOP = new SqlHopTableFunction();
public static final SqlFunction CUMULATE = new SqlCumulateTableFunction();
+ public static final SqlFunction SESSION = new SqlSessionTableFunction();
// Catalog Functions
public static final SqlFunction CURRENT_DATABASE =
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/SqlSessionTableFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/SqlSessionTableFunction.java
new file mode 100644
index 00000000000..5e5f62268f7
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/SqlSessionTableFunction.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.functions.sql;
+
+import org.apache.flink.shaded.guava32.com.google.common.collect.ImmutableList;
+import org.apache.flink.shaded.guava32.com.google.common.collect.ImmutableMap;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.TableCharacteristic;
+import org.apache.calcite.sql.type.SqlTypeUtil;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.util.Map;
+
+/**
+ * SqlSessionTableFunction implements an operator for per-key sessionization. It allows four
+ * parameters:
+ *
+ * <ol>
+ * <li>table as data source
+ * <li>a descriptor to provide a watermarked column name from the input table
+ * <li>a descriptor to provide a column as key, on which sessionization will be applied, optional
+ * <li>an interval parameter to specify a inactive activity gap to break sessions
+ * </ol>
+ *
+ * <p>Mainly copy from {@link org.apache.calcite.sql.SqlSessionTableFunction}. FLINK modifications
+ * are the following:
+ *
+ * <ol>
+ * <li>Add some attributes required for PTF.
+ * <li>Invalid the deprecated syntax in calcite about 'partition by' in window tvf like: (TABLE
+ * table_name, DESCRIPTOR(timecol), DESCRIPTOR(key) optional, datetime interval)
+ * </ol>
+ */
+public class SqlSessionTableFunction extends SqlWindowTableFunction {
+ public SqlSessionTableFunction() {
+ super(SqlKind.SESSION.name(), new OperandMetadataImpl());
+ }
+
+ private final Map<Integer, TableCharacteristic> tableParams =
+ ImmutableMap.of(
+ 0,
+ TableCharacteristic.builder(TableCharacteristic.Semantics.SET)
+ .pruneIfEmpty()
+ .passColumnsThrough()
+ .build());
+
+ /** Operand type checker for SESSION. */
+ private static class OperandMetadataImpl extends AbstractOperandMetadata {
+ OperandMetadataImpl() {
+ super(ImmutableList.of(PARAM_DATA, PARAM_TIMECOL, PARAM_SIZE), 3);
+ }
+
+ @Override
+ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) {
+ if (!checkTableAndDescriptorOperands(callBinding, 1)) {
+ return throwValidationSignatureErrorOrReturnFalse(callBinding, throwOnFailure);
+ }
+
+ final SqlValidator validator = callBinding.getValidator();
+ final SqlNode operand2 = callBinding.operand(2);
+ final RelDataType type2 = validator.getValidatedNodeType(operand2);
+ if (!SqlTypeUtil.isInterval(type2)) {
+ return throwValidationSignatureErrorOrReturnFalse(callBinding, throwOnFailure);
+ }
+
+ return throwExceptionOrReturnFalse(
+ checkTimeColumnDescriptorOperand(callBinding, 1), throwOnFailure);
+ }
+
+ @Override
+ public String getAllowedSignatures(SqlOperator op, String opName) {
+ return opName
+ + "(TABLE table_name [PARTITION BY (keycols, ...)], "
+ + "DESCRIPTOR(timecol), datetime interval)";
+ }
+ }
+
+ @Override
+ public @Nullable TableCharacteristic tableCharacteristic(int ordinal) {
+ return tableParams.get(ordinal);
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/CumulativeWindowSpec.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/CumulativeWindowSpec.java
index dc4b553ed10..02055e70ecf 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/CumulativeWindowSpec.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/CumulativeWindowSpec.java
@@ -60,7 +60,7 @@ public class CumulativeWindowSpec implements WindowSpec {
}
@Override
- public String toSummaryString(String windowing) {
+ public String toSummaryString(String windowing, String[] inputFieldNames) {
if (offset == null) {
return String.format(
"CUMULATE(%s, max_size=[%s], step=[%s])",
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/HoppingWindowSpec.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/HoppingWindowSpec.java
index 7b933879afd..fd2a04cae05 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/HoppingWindowSpec.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/HoppingWindowSpec.java
@@ -60,7 +60,7 @@ public class HoppingWindowSpec implements WindowSpec {
}
@Override
- public String toSummaryString(String windowing) {
+ public String toSummaryString(String windowing, String[] inputFieldNames) {
if (offset == null) {
return String.format(
"HOP(%s, size=[%s], slide=[%s])",
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/SessionWindowSpec.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/SessionWindowSpec.java
new file mode 100644
index 00000000000..dd397d5cbec
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/SessionWindowSpec.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.plan.logical;
+
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeName;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.IntStream;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.TimeUtils.formatWithHighestUnit;
+
+/** Logical representation of a session window specification. */
+@JsonTypeName("SessionWindow")
+public class SessionWindowSpec implements WindowSpec {
+ public static final String FIELD_NAME_GAP = "gap";
+ public static final String FIELD_NAME_PARTITION_KEYS = "partition_key_indices";
+
+ @JsonProperty(FIELD_NAME_GAP)
+ private final Duration gap;
+
+ @JsonProperty(FIELD_NAME_PARTITION_KEYS)
+ private final int[] partitionKeyIndices;
+
+ @JsonCreator
+ public SessionWindowSpec(
+ @JsonProperty(FIELD_NAME_GAP) Duration gap,
+ @JsonProperty(FIELD_NAME_PARTITION_KEYS) int[] partitionKeyIndices) {
+ this.gap = checkNotNull(gap);
+ this.partitionKeyIndices = checkNotNull(partitionKeyIndices);
+ }
+
+ @Override
+ public String toSummaryString(String windowing, String[] inputFieldNames) {
+ if (partitionKeyIndices.length == 0) {
+ return String.format("SESSION(%s, gap=[%s])", windowing, formatWithHighestUnit(gap));
+ } else {
+ String[] partitionKeyNames =
+ IntStream.of(partitionKeyIndices)
+ .mapToObj(idx -> inputFieldNames[idx])
+ .toArray(String[]::new);
+ return String.format(
+ "SESSION(%s, gap=[%s], partition keys=%s)",
+ windowing, formatWithHighestUnit(gap), Arrays.toString(partitionKeyNames));
+ }
+ }
+
+ public Duration getGap() {
+ return gap;
+ }
+
+ public int[] getPartitionKeyIndices() {
+ return partitionKeyIndices;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SessionWindowSpec that = (SessionWindowSpec) o;
+ return Objects.equals(gap, that.gap)
+ && Arrays.equals(partitionKeyIndices, that.partitionKeyIndices);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(SessionWindowSpec.class, gap, Arrays.hashCode(partitionKeyIndices));
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "SESSION(gap=[%s],partitionKeys=%s)",
+ formatWithHighestUnit(gap), Arrays.toString(partitionKeyIndices));
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/SliceAttachedWindowingStrategy.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/SliceAttachedWindowingStrategy.java
index f1a9cf87f7b..cc61d215fd4 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/SliceAttachedWindowingStrategy.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/SliceAttachedWindowingStrategy.java
@@ -54,7 +54,7 @@ public class SliceAttachedWindowingStrategy extends WindowingStrategy {
public String toSummaryString(String[] inputFieldNames) {
checkArgument(sliceEnd < inputFieldNames.length);
String windowing = String.format("slice_end=[%s]", inputFieldNames[sliceEnd]);
- return window.toSummaryString(windowing);
+ return window.toSummaryString(windowing, inputFieldNames);
}
public int getSliceEnd() {
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/TimeAttributeWindowingStrategy.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/TimeAttributeWindowingStrategy.java
index 6a60ce4d16e..f2fc9badc11 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/TimeAttributeWindowingStrategy.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/TimeAttributeWindowingStrategy.java
@@ -47,7 +47,7 @@ public class TimeAttributeWindowingStrategy extends WindowingStrategy {
public String toSummaryString(String[] inputFieldNames) {
checkArgument(timeAttributeIndex >= 0 && timeAttributeIndex < inputFieldNames.length);
String windowing = String.format("time_col=[%s]", inputFieldNames[timeAttributeIndex]);
- return window.toSummaryString(windowing);
+ return window.toSummaryString(windowing, inputFieldNames);
}
public int getTimeAttributeIndex() {
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/TumblingWindowSpec.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/TumblingWindowSpec.java
index 5373c86e8c6..fdf5e92649a 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/TumblingWindowSpec.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/TumblingWindowSpec.java
@@ -54,7 +54,7 @@ public class TumblingWindowSpec implements WindowSpec {
}
@Override
- public String toSummaryString(String windowing) {
+ public String toSummaryString(String windowing, String[] inputFieldNames) {
if (offset == null) {
return String.format("TUMBLE(%s, size=[%s])", windowing, formatWithHighestUnit(size));
} else {
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/WindowAttachedWindowingStrategy.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/WindowAttachedWindowingStrategy.java
index cb3cf7c3f61..e3c5628ef97 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/WindowAttachedWindowingStrategy.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/WindowAttachedWindowingStrategy.java
@@ -77,7 +77,7 @@ public class WindowAttachedWindowingStrategy extends WindowingStrategy {
"win_start=[%s], win_end=[%s]",
inputFieldNames[windowStart], inputFieldNames[windowEnd]);
}
- return window.toSummaryString(windowing);
+ return window.toSummaryString(windowing, inputFieldNames);
}
public int getWindowStart() {
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/WindowSpec.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/WindowSpec.java
index bb3c261f78b..057be4254fd 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/WindowSpec.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/logical/WindowSpec.java
@@ -26,9 +26,10 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTyp
@JsonSubTypes({
@JsonSubTypes.Type(value = TumblingWindowSpec.class),
@JsonSubTypes.Type(value = HoppingWindowSpec.class),
- @JsonSubTypes.Type(value = CumulativeWindowSpec.class)
+ @JsonSubTypes.Type(value = CumulativeWindowSpec.class),
+ @JsonSubTypes.Type(value = SessionWindowSpec.class)
})
public interface WindowSpec {
- String toSummaryString(String windowing);
+ String toSummaryString(String windowing, String[] inputFieldNames);
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecWindowTableFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecWindowTableFunction.java
index 0319597004c..2d9924e9f97 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecWindowTableFunction.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/common/CommonExecWindowTableFunction.java
@@ -20,8 +20,10 @@ package org.apache.flink.table.planner.plan.nodes.exec.common;
import org.apache.flink.api.dag.Transformation;
import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.table.api.TableException;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.planner.delegation.PlannerBase;
+import org.apache.flink.table.planner.plan.logical.SessionWindowSpec;
import org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy;
import org.apache.flink.table.planner.plan.nodes.exec.ExecEdge;
import org.apache.flink.table.planner.plan.nodes.exec.ExecNode;
@@ -77,6 +79,11 @@ public abstract class CommonExecWindowTableFunction extends ExecNodeBase<RowData
@Override
protected Transformation<RowData> translateToPlanInternal(
PlannerBase planner, ExecNodeConfig config) {
+ if (windowingStrategy.getWindow() instanceof SessionWindowSpec) {
+ // TODO Support session window table function in ExecWindowTableFunction. See
+ // more at FLINK-34100
+ throw new TableException("Session Window TableFunction is not supported yet.");
+ }
final ExecEdge inputEdge = getInputEdges().get(0);
final Transformation<RowData> inputTransform =
(Transformation<RowData>) inputEdge.translateToPlan(planner);
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregate.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregate.java
index b8c63d6dde6..fafc27b20da 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregate.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregate.java
@@ -25,9 +25,14 @@ import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.SimpleOperatorFactory;
import org.apache.flink.streaming.api.transformations.OneInputTransformation;
import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.expressions.FieldReferenceExpression;
import org.apache.flink.table.planner.codegen.CodeGeneratorContext;
import org.apache.flink.table.planner.codegen.agg.AggsHandlerCodeGenerator;
import org.apache.flink.table.planner.delegation.PlannerBase;
+import org.apache.flink.table.planner.plan.logical.LogicalWindow;
+import org.apache.flink.table.planner.plan.logical.SessionGroupWindow;
+import org.apache.flink.table.planner.plan.logical.SessionWindowSpec;
+import org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy;
import org.apache.flink.table.planner.plan.logical.WindowingStrategy;
import org.apache.flink.table.planner.plan.nodes.exec.ExecEdge;
import org.apache.flink.table.planner.plan.nodes.exec.ExecNode;
@@ -44,6 +49,7 @@ import org.apache.flink.table.planner.utils.TableConfigUtils;
import org.apache.flink.table.runtime.generated.GeneratedNamespaceAggsHandleFunction;
import org.apache.flink.table.runtime.groupwindow.NamedWindowProperty;
import org.apache.flink.table.runtime.groupwindow.WindowProperty;
+import org.apache.flink.table.runtime.groupwindow.WindowReference;
import org.apache.flink.table.runtime.keyselector.RowDataKeySelector;
import org.apache.flink.table.runtime.operators.aggregate.window.SlicingWindowAggOperatorBuilder;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigner;
@@ -54,6 +60,7 @@ import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
import org.apache.flink.table.runtime.util.TimeWindowUtil;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.Preconditions;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
@@ -69,6 +76,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import static org.apache.flink.table.expressions.ApiExpressionUtils.intervalOfMillis;
+import static org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter.fromLogicalTypeToDataType;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
@@ -160,6 +169,12 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
@Override
protected Transformation<RowData> translateToPlanInternal(
PlannerBase planner, ExecNodeConfig config) {
+ // TODO Currently, the operator of WindowAggregate does not support Session Window, and it
+ // needs to fall back to the legacy GroupWindowAggregate. See more at FLINK-34048.
+ if (windowing.getWindow() instanceof SessionWindowSpec) {
+ return fallbackToLegacyGroupWindowAggregate(planner, config);
+ }
+
final ExecEdge inputEdge = getInputEdges().get(0);
final Transformation<RowData> inputTransform =
(Transformation<RowData>) inputEdge.translateToPlan(planner);
@@ -262,4 +277,47 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
sliceAssigner,
shiftTimeZone);
}
+
+ private Transformation<RowData> fallbackToLegacyGroupWindowAggregate(
+ PlannerBase planner, ExecNodeConfig config) {
+ Preconditions.checkState(windowing.getWindow() instanceof SessionWindowSpec);
+
+ if (windowing instanceof TimeAttributeWindowingStrategy) {
+ LogicalType timeAttributeType = windowing.getTimeAttributeType();
+ LogicalWindow logicalWindow =
+ new SessionGroupWindow(
+ new WindowReference("w$", timeAttributeType),
+ new FieldReferenceExpression(
+ // mock an empty time field name here
+ "",
+ fromLogicalTypeToDataType(timeAttributeType),
+ 0,
+ ((TimeAttributeWindowingStrategy) windowing)
+ .getTimeAttributeIndex()),
+ intervalOfMillis(
+ ((SessionWindowSpec) windowing.getWindow())
+ .getGap()
+ .toMillis()));
+
+ StreamExecGroupWindowAggregate groupWindowAggregate =
+ new StreamExecGroupWindowAggregate(
+ planner.getTableConfig(),
+ grouping,
+ aggCalls,
+ logicalWindow,
+ namedWindowProperties,
+ needRetraction,
+ InputProperty.DEFAULT,
+ (RowType) getOutputType(),
+ getDescription());
+
+ groupWindowAggregate.setInputEdges(getInputEdges());
+ return groupWindowAggregate.translateToPlanInternal(planner, config);
+ }
+
+ throw new UnsupportedOperationException(
+ String.format(
+ "Unsupported windowing strategy: %s for session window.",
+ windowing.getClass()));
+ }
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/ProjectWindowTableFunctionTransposeRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/ProjectWindowTableFunctionTransposeRule.java
index 8c896e8bf15..8d6ab6ddd8d 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/ProjectWindowTableFunctionTransposeRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/logical/ProjectWindowTableFunctionTransposeRule.java
@@ -19,7 +19,9 @@
package org.apache.flink.table.planner.plan.rules.logical;
import org.apache.flink.table.planner.calcite.FlinkTypeFactory;
+import org.apache.flink.table.planner.calcite.RexSetSemanticsTableCall;
import org.apache.flink.table.planner.functions.sql.SqlWindowTableFunction;
+import org.apache.flink.table.planner.plan.logical.SessionWindowSpec;
import org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy;
import org.apache.flink.table.planner.plan.utils.WindowUtil;
import org.apache.flink.table.types.logical.LogicalType;
@@ -41,6 +43,7 @@ import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Pair;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -76,8 +79,7 @@ public class ProjectWindowTableFunctionTransposeRule extends RelOptRule {
LogicalTableFunctionScan scan = call.rel(1);
RelNode scanInput = scan.getInput(0);
TimeAttributeWindowingStrategy windowingStrategy =
- WindowUtil.convertToWindowingStrategy(
- (RexCall) scan.getCall(), scanInput.getRowType());
+ WindowUtil.convertToWindowingStrategy((RexCall) scan.getCall(), scanInput);
// 1. get fields to push down
ImmutableBitSet projectFields = RelOptUtil.InputFinder.bits(project.getProjects(), null);
int scanInputFieldCount = scanInput.getRowType().getFieldCount();
@@ -85,6 +87,12 @@ public class ProjectWindowTableFunctionTransposeRule extends RelOptRule {
ImmutableBitSet.range(0, scanInputFieldCount)
.intersect(projectFields)
.set(windowingStrategy.getTimeAttributeIndex());
+ // partition keys in session window tvf also need be pushed down
+ if (windowingStrategy.getWindow() instanceof SessionWindowSpec) {
+ SessionWindowSpec sessionWindowSpec = (SessionWindowSpec) windowingStrategy.getWindow();
+ int[] partitionKeyIndices = sessionWindowSpec.getPartitionKeyIndices();
+ toPushFields = toPushFields.union(ImmutableBitSet.of(partitionKeyIndices));
+ }
if (toPushFields.cardinality() == scanInputFieldCount) {
return;
}
@@ -169,6 +177,14 @@ public class ProjectWindowTableFunctionTransposeRule extends RelOptRule {
for (RexNode next : windowCall.getOperands()) {
newOperands.add(adjustInputRef(next, mapping));
}
+ if (windowCall instanceof RexSetSemanticsTableCall) {
+ RexSetSemanticsTableCall originalCall = (RexSetSemanticsTableCall) windowCall;
+ int[] newPartitionKeys =
+ Arrays.stream(originalCall.getPartitionKeys()).map(mapping::get).toArray();
+ int[] newOrderKeys =
+ Arrays.stream(originalCall.getOrderKeys()).map(mapping::get).toArray();
+ return originalCall.copy(newOperands, newPartitionKeys, newOrderKeys);
+ }
return relBuilder.call(windowCall.getOperator(), newOperands);
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/TwoStageOptimizedWindowAggregateRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/TwoStageOptimizedWindowAggregateRule.java
index 70d7aaac013..0d34dddad7e 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/TwoStageOptimizedWindowAggregateRule.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/rules/physical/stream/TwoStageOptimizedWindowAggregateRule.java
@@ -19,6 +19,7 @@
package org.apache.flink.table.planner.plan.rules.physical.stream;
import org.apache.flink.table.api.TableConfig;
+import org.apache.flink.table.planner.plan.logical.SessionWindowSpec;
import org.apache.flink.table.planner.plan.logical.SliceAttachedWindowingStrategy;
import org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy;
import org.apache.flink.table.planner.plan.logical.WindowAttachedWindowingStrategy;
@@ -93,6 +94,12 @@ public class TwoStageOptimizedWindowAggregateRule extends RelOptRule {
return false;
}
+ // session window doesn't support two-phase,
+ // otherwise window assigner results may be different
+ if (windowing.getWindow() instanceof SessionWindowSpec) {
+ return false;
+ }
+
// all aggregate function should support merge() method
if (!AggregateUtil.doAllSupportPartialMerge(windowAgg.aggInfoList().aggInfos())) {
return false;
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/trait/RelWindowProperties.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/trait/RelWindowProperties.java
index fc417eb86f6..1966234a9b4 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/trait/RelWindowProperties.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/trait/RelWindowProperties.java
@@ -93,6 +93,19 @@ public class RelWindowProperties {
this.timeAttributeType);
}
+ public @Nullable RelWindowProperties copy(
+ ImmutableBitSet windowStartColumns,
+ ImmutableBitSet windowEndColumns,
+ ImmutableBitSet windowTimeColumns,
+ WindowSpec windowSpec) {
+ return create(
+ windowStartColumns,
+ windowEndColumns,
+ windowTimeColumns,
+ windowSpec,
+ this.timeAttributeType);
+ }
+
public ImmutableBitSet getWindowStartColumns() {
return windowStartColumns;
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdWindowProperties.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdWindowProperties.scala
index a688302e914..80cce27a559 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdWindowProperties.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/metadata/FlinkRelMdWindowProperties.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.metadata
import org.apache.flink.table.planner.{JArrayList, JHashMap, JList}
import org.apache.flink.table.planner.plan.`trait`.RelWindowProperties
-import org.apache.flink.table.planner.plan.logical.WindowSpec
+import org.apache.flink.table.planner.plan.logical.{SessionWindowSpec, WindowSpec}
import org.apache.flink.table.planner.plan.nodes.calcite.{Expand, WatermarkAssigner}
import org.apache.flink.table.planner.plan.nodes.logical.{FlinkLogicalAggregate, FlinkLogicalCorrelate, FlinkLogicalJoin, FlinkLogicalRank}
import org.apache.flink.table.planner.plan.nodes.physical.common.CommonPhysicalLookupJoin
@@ -29,6 +29,7 @@ import org.apache.flink.table.planner.plan.utils.WindowJoinUtil.satisfyWindowJoi
import org.apache.flink.table.planner.plan.utils.WindowUtil.{convertToWindowingStrategy, groupingContainsWindowStartEnd, isWindowTableFunctionCall}
import org.apache.flink.table.runtime.groupwindow._
import org.apache.flink.table.types.logical.LogicalType
+import org.apache.flink.util.Preconditions
import org.apache.calcite.plan.hep.HepRelVertex
import org.apache.calcite.plan.volcano.RelSubset
@@ -92,13 +93,56 @@ class FlinkRelMdWindowProperties private extends MetadataHandler[FlinkMetadata.W
return null
}
+ val windowSpec = childProps.getWindowSpec
+ if (!validWindowProperties(windowSpec, mapInToOutPos)) {
+ // If the window becomes illegal after passing through calc or project,
+ // return no window properties
+ return null
+ }
+
childProps.copy(
transformColumnIndex(childProps.getWindowStartColumns, mapInToOutPos),
transformColumnIndex(childProps.getWindowEndColumns, mapInToOutPos),
- transformColumnIndex(childProps.getWindowTimeColumns, mapInToOutPos)
+ transformColumnIndex(childProps.getWindowTimeColumns, mapInToOutPos),
+ updateWindowSpec(windowSpec, mapInToOutPos)
)
}
+ /**
+ * Validate the window properties when passing through calc or project.
+ *
+ * This method only checks the window like session window that contains partition keys. See more
+ * at [[WindowUtil.validatePartitionKeyIfNecessary]]
+ */
+ private def validWindowProperties(
+ windowSpec: WindowSpec,
+ mapInToOutPos: JHashMap[Int, JList[Int]]): Boolean = {
+ windowSpec match {
+ case session: SessionWindowSpec =>
+ val oldPartitionKeys = session.getPartitionKeyIndices
+ oldPartitionKeys.forall(oldPartitionKey => mapInToOutPos.contains(oldPartitionKey))
+ case _ => true
+ }
+ }
+
+ /** Update partition key indices in session window spec according to the projection map. */
+ private def updateWindowSpec(
+ oldWindowSpec: WindowSpec,
+ mapInToOutPos: JHashMap[Int, JList[Int]]): WindowSpec = {
+ oldWindowSpec match {
+ case session: SessionWindowSpec =>
+ val oldPartitionKeys = session.getPartitionKeyIndices
+ val newPartitionKeys = oldPartitionKeys.map(
+ oldPartitionKey => {
+ val newPartitionKey = mapInToOutPos.get(oldPartitionKey)
+ Preconditions.checkArgument(newPartitionKey.size == 1)
+ newPartitionKey.head
+ })
+ new SessionWindowSpec(session.getGap, newPartitionKeys)
+ case _ => oldWindowSpec
+ }
+ }
+
private def transformColumnIndex(
columns: ImmutableBitSet,
mapping: JHashMap[Int, JList[Int]]): ImmutableBitSet = {
@@ -196,7 +240,7 @@ class FlinkRelMdWindowProperties private extends MetadataHandler[FlinkMetadata.W
if (isWindowTableFunctionCall(rel.getCall)) {
val fieldCount = rel.getRowType.getFieldCount
val windowingStrategy =
- convertToWindowingStrategy(rel.getCall.asInstanceOf[RexCall], rel.getInput(0).getRowType)
+ convertToWindowingStrategy(rel.getCall.asInstanceOf[RexCall], rel.getInput(0))
RelWindowProperties.create(
ImmutableBitSet.of(fieldCount - 3),
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/common/CommonPhysicalWindowTableFunction.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/common/CommonPhysicalWindowTableFunction.scala
index 7d01d226d71..f0bc89c401a 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/common/CommonPhysicalWindowTableFunction.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/common/CommonPhysicalWindowTableFunction.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.planner.plan.nodes.common
import org.apache.flink.table.api.TableException
-import org.apache.flink.table.planner.plan.logical.{CumulativeWindowSpec, HoppingWindowSpec, TimeAttributeWindowingStrategy, TumblingWindowSpec}
+import org.apache.flink.table.planner.plan.logical.{CumulativeWindowSpec, HoppingWindowSpec, SessionWindowSpec, TimeAttributeWindowingStrategy, TumblingWindowSpec}
import org.apache.calcite.plan.{RelOptCluster, RelTraitSet}
import org.apache.calcite.rel.`type`.RelDataType
@@ -58,6 +58,8 @@ abstract class CommonPhysicalWindowTableFunction(
val maxWindowsNum = (cumulateWindow.getMaxSize.toMillis /
cumulateWindow.getStep.toMillis).asInstanceOf[Int]
childRowCnt * maxWindowsNum
+ case sessionWindow: SessionWindowSpec =>
+ childRowCnt * sessionWindow.getGap.toMillis.asInstanceOf[Int]
case windowSpec =>
throw new TableException(s"Unknown window spec: ${windowSpec.getClass.getSimpleName}")
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalLocalWindowAggregate.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalLocalWindowAggregate.scala
index a094dcaff5e..49e73fede83 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalLocalWindowAggregate.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamPhysicalLocalWindowAggregate.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.planner.plan.nodes.physical.stream
import org.apache.flink.table.planner.calcite.FlinkTypeFactory
-import org.apache.flink.table.planner.plan.logical.{TimeAttributeWindowingStrategy, WindowAttachedWindowingStrategy, WindowingStrategy}
+import org.apache.flink.table.planner.plan.logical.{SessionWindowSpec, TimeAttributeWindowingStrategy, WindowAttachedWindowingStrategy, WindowingStrategy}
import org.apache.flink.table.planner.plan.nodes.exec.{ExecNode, InputProperty}
import org.apache.flink.table.planner.plan.nodes.exec.stream.StreamExecLocalWindowAggregate
import org.apache.flink.table.planner.plan.rules.physical.stream.TwoStageOptimizedWindowAggregateRule
@@ -74,8 +74,18 @@ class StreamPhysicalLocalWindowAggregate(
override def isValid(litmus: Litmus, context: RelNode.Context): Boolean = {
windowing match {
- case _: WindowAttachedWindowingStrategy | _: TimeAttributeWindowingStrategy =>
+ case _: WindowAttachedWindowingStrategy =>
// pass
+ case tws: TimeAttributeWindowingStrategy =>
+ tws.getWindow match {
+ case _: SessionWindowSpec =>
+ return litmus.fail(
+ "StreamPhysicalLocalWindowAggregate should not accept " +
+ "TimeAttributeWindowingStrategy with Session window. " +
+ "This should never happen, please open an issue.")
+ case _ =>
+ // pass
+ }
case _ =>
return litmus.fail(
"StreamPhysicalLocalWindowAggregate should only accepts " +
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/SplitAggregateRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/SplitAggregateRule.scala
index 5ca5526059d..61827342614 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/SplitAggregateRule.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/SplitAggregateRule.scala
@@ -22,6 +22,7 @@ import org.apache.flink.table.api.config.OptimizerConfigOptions
import org.apache.flink.table.planner.calcite.{FlinkLogicalRelFactories, FlinkRelBuilder}
import org.apache.flink.table.planner.functions.sql.{FlinkSqlOperatorTable, SqlFirstLastValueAggFunction}
import org.apache.flink.table.planner.plan.PartialFinalType
+import org.apache.flink.table.planner.plan.logical.SessionWindowSpec
import org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery
import org.apache.flink.table.planner.plan.nodes.FlinkRelNode
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalAggregate
@@ -125,11 +126,16 @@ class SplitAggregateRule
val windowProps = fmq.getRelWindowProperties(agg.getInput)
val isWindowAgg = WindowUtil.groupingContainsWindowStartEnd(agg.getGroupSet, windowProps)
val isProctimeWindowAgg = isWindowAgg && !windowProps.isRowtime
+ // disable distinct split for session window,
+ // otherwise window assigner results may be different
+ val isSessionWindowAgg = isWindowAgg &&
+ windowProps.getWindowSpec.isInstanceOf[SessionWindowSpec]
// TableAggregate is not supported. see also FLINK-21923.
val isTableAgg = AggregateUtil.isTableAggregate(agg.getAggCallList)
agg.partialFinalType == PartialFinalType.NONE && agg.containsDistinctCall() &&
- splitDistinctAggEnabled && isAllAggSplittable && !isProctimeWindowAgg && !isTableAgg
+ splitDistinctAggEnabled && isAllAggSplittable && !isProctimeWindowAgg &&
+ !isTableAgg && !isSessionWindowAgg
}
override def onMatch(call: RelOptRuleCall): Unit = {
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalWindowTableFunctionRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalWindowTableFunctionRule.scala
index c8d90a8cca2..c032cd27ce3 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalWindowTableFunctionRule.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/batch/BatchPhysicalWindowTableFunctionRule.scala
@@ -50,7 +50,7 @@ class BatchPhysicalWindowTableFunctionRule(config: Config) extends ConverterRule
traitSet,
newInput,
scan.getRowType,
- convertToWindowingStrategy(scan.getCall.asInstanceOf[RexCall], newInput.getRowType)
+ convertToWindowingStrategy(scan.getCall.asInstanceOf[RexCall], newInput)
)
}
}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/PullUpWindowTableFunctionIntoWindowAggregateRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/PullUpWindowTableFunctionIntoWindowAggregateRule.scala
index 8ae5d3cd886..f98252eed98 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/PullUpWindowTableFunctionIntoWindowAggregateRule.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/PullUpWindowTableFunctionIntoWindowAggregateRule.scala
@@ -18,19 +18,24 @@
package org.apache.flink.table.planner.plan.rules.physical.stream
import org.apache.flink.table.planner.plan.`trait`.FlinkRelDistribution
-import org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy
+import org.apache.flink.table.planner.plan.logical.{SessionWindowSpec, TimeAttributeWindowingStrategy, WindowSpec}
import org.apache.flink.table.planner.plan.metadata.FlinkRelMetadataQuery
import org.apache.flink.table.planner.plan.nodes.FlinkConventions
import org.apache.flink.table.planner.plan.nodes.physical.stream.{StreamPhysicalCalc, StreamPhysicalExchange, StreamPhysicalWindowAggregate, StreamPhysicalWindowTableFunction}
import org.apache.flink.table.planner.plan.utils.WindowUtil
import org.apache.flink.table.planner.plan.utils.WindowUtil.buildNewProgramWithoutWindowColumns
+import org.apache.flink.util.Preconditions.checkArgument
import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall}
import org.apache.calcite.plan.RelOptRule.{any, operand}
+import org.apache.calcite.plan.volcano.RelSubset
import org.apache.calcite.rel.{RelCollations, RelNode}
+import org.apache.calcite.rex.RexInputRef
import org.apache.calcite.util.ImmutableBitSet
+import scala.annotation.tailrec
import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
/**
* Planner rule that tries to pull up [[StreamPhysicalWindowTableFunction]] into a
@@ -73,7 +78,12 @@ class PullUpWindowTableFunctionIntoWindowAggregateRule
val windowTVF: StreamPhysicalWindowTableFunction = call.rel(3)
val fmq = FlinkRelMetadataQuery.reuseOrCreate(windowAgg.getCluster.getMetadataQuery)
val cluster = windowAgg.getCluster
- val input = windowTVF.getInput
+ val input = unwrapRel(windowTVF.getInput) match {
+ case exchange: StreamPhysicalExchange =>
+ exchange.getInput
+ case other =>
+ other
+ }
val inputRowType = input.getRowType
val requiredInputTraitSet = input.getTraitSet.replace(FlinkConventions.STREAM_PHYSICAL)
@@ -115,8 +125,9 @@ class PullUpWindowTableFunctionIntoWindowAggregateRule
// -----------------------------------------------------------------------------
// 3. Adjust aggregate arguments index and construct new window aggregate node
// -----------------------------------------------------------------------------
+ val newWindowSpec = updateWindowSpec(windowTVF.windowing.getWindow, newCalc)
val newWindowing = new TimeAttributeWindowingStrategy(
- windowTVF.windowing.getWindow,
+ newWindowSpec,
windowTVF.windowing.getTimeAttributeType,
timeAttributeIndex)
val providedTraitSet = windowAgg.getTraitSet.replace(FlinkConventions.STREAM_PHYSICAL)
@@ -146,6 +157,47 @@ class PullUpWindowTableFunctionIntoWindowAggregateRule
call.transformTo(newWindowAgg)
}
+
+ @tailrec
+ private def unwrapRel(rel: RelNode): RelNode = {
+ rel match {
+ case relSubset: RelSubset =>
+ unwrapRel(relSubset.getOriginal)
+ case _ =>
+ rel
+ }
+ }
+
+ private def updateWindowSpec(oldWindowSpec: WindowSpec, calc: StreamPhysicalCalc): WindowSpec = {
+ oldWindowSpec match {
+ case sessionWindowSpec: SessionWindowSpec =>
+ val windowPartitionKeys = sessionWindowSpec.getPartitionKeyIndices
+ val newPartitionKeysThroughCalc =
+ getSessionPartitionKeysThroughCalc(windowPartitionKeys, calc)
+ checkArgument(windowPartitionKeys.length == newPartitionKeysThroughCalc.length)
+ new SessionWindowSpec(sessionWindowSpec.getGap, newPartitionKeysThroughCalc)
+ case _ => oldWindowSpec
+ }
+ }
+
+ private def getSessionPartitionKeysThroughCalc(
+ sessionWindowPartitionKeyIndices: Array[Int],
+ calc: StreamPhysicalCalc): Array[Int] = {
+ val newPartitionKeyIndices = ArrayBuffer[Int]()
+ val program = calc.getProgram
+ program.getNamedProjects.zipWithIndex.foreach {
+ case (project, index) =>
+ val expr = program.expandLocalRef(project.left)
+ expr match {
+ case inputRef: RexInputRef =>
+ if (sessionWindowPartitionKeyIndices.indexOf(inputRef.getIndex) != -1) {
+ newPartitionKeyIndices += index
+ }
+ case _ => // ignore
+ }
+ }
+ newPartitionKeyIndices.toArray
+ }
}
object PullUpWindowTableFunctionIntoWindowAggregateRule {
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalWindowTableFunctionRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalWindowTableFunctionRule.scala
index 6ceeef69ac1..1b9176a0b8b 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalWindowTableFunctionRule.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamPhysicalWindowTableFunctionRule.scala
@@ -17,13 +17,16 @@
*/
package org.apache.flink.table.planner.plan.rules.physical.stream
+import org.apache.flink.table.planner.calcite.RexSetSemanticsTableCall
+import org.apache.flink.table.planner.functions.sql.FlinkSqlOperatorTable
+import org.apache.flink.table.planner.plan.`trait`.FlinkRelDistribution
import org.apache.flink.table.planner.plan.nodes.FlinkConventions
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalTableFunctionScan
import org.apache.flink.table.planner.plan.nodes.physical.stream.StreamPhysicalWindowTableFunction
import org.apache.flink.table.planner.plan.utils.WindowUtil
import org.apache.flink.table.planner.plan.utils.WindowUtil.{convertToWindowingStrategy, validateTimeFieldWithTimeAttribute}
-import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet}
+import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall}
import org.apache.calcite.rel.RelNode
import org.apache.calcite.rel.convert.ConverterRule
import org.apache.calcite.rel.convert.ConverterRule.Config
@@ -42,8 +45,14 @@ class StreamPhysicalWindowTableFunctionRule(config: Config) extends ConverterRul
def convert(rel: RelNode): RelNode = {
val scan: FlinkLogicalTableFunctionScan = rel.asInstanceOf[FlinkLogicalTableFunctionScan]
- val traitSet: RelTraitSet = rel.getTraitSet.replace(FlinkConventions.STREAM_PHYSICAL)
- val newInput = RelOptRule.convert(scan.getInput(0), FlinkConventions.STREAM_PHYSICAL)
+
+ val requiredDistribution = getDistribution(scan.getCall.asInstanceOf[RexCall])
+ val requiredTraitSet = rel.getCluster.getPlanner
+ .emptyTraitSet()
+ .replace(requiredDistribution)
+ .replace(FlinkConventions.STREAM_PHYSICAL)
+ val providedTraitSet = rel.getTraitSet.replace(FlinkConventions.STREAM_PHYSICAL)
+ val newInput = RelOptRule.convert(scan.getInput(0), requiredTraitSet)
val windowTableFunction = scan.getCall.asInstanceOf[RexCall]
val inputRowType = newInput.getRowType
@@ -51,12 +60,27 @@ class StreamPhysicalWindowTableFunctionRule(config: Config) extends ConverterRul
validateTimeFieldWithTimeAttribute(windowTableFunction, inputRowType)
new StreamPhysicalWindowTableFunction(
scan.getCluster,
- traitSet,
+ providedTraitSet,
newInput,
scan.getRowType,
- convertToWindowingStrategy(windowTableFunction, inputRowType)
+ convertToWindowingStrategy(windowTableFunction, newInput)
)
}
+
+ private def getDistribution(windowCall: RexCall): FlinkRelDistribution = {
+ windowCall.getOperator match {
+ case FlinkSqlOperatorTable.SESSION =>
+ windowCall match {
+ case setSemanticsTableCall: RexSetSemanticsTableCall =>
+ val partitionKeys = setSemanticsTableCall.getPartitionKeys
+ FlinkRelDistribution.hash(partitionKeys, requireStrict = true)
+ case _ =>
+ FlinkRelDistribution.SINGLETON
+ }
+ case _ =>
+ FlinkRelDistribution.DEFAULT
+ }
+ }
}
object StreamPhysicalWindowTableFunctionRule {
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
index ed06bff2824..995a89a8a03 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/plan/utils/WindowUtil.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.planner.plan.utils
import org.apache.flink.table.api.{DataTypes, TableConfig, TableException, ValidationException}
import org.apache.flink.table.planner.JBigDecimal
-import org.apache.flink.table.planner.calcite.FlinkTypeFactory
+import org.apache.flink.table.planner.calcite.{FlinkTypeFactory, RexSetSemanticsTableCall}
import org.apache.flink.table.planner.functions.sql.{FlinkSqlOperatorTable, SqlWindowTableFunction}
import org.apache.flink.table.planner.plan.`trait`.RelWindowProperties
import org.apache.flink.table.planner.plan.logical._
@@ -36,7 +36,7 @@ import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.canBeTimeAtt
import org.apache.calcite.plan.volcano.RelSubset
import org.apache.calcite.rel.`type`.RelDataType
import org.apache.calcite.rel.{RelNode, SingleRel}
-import org.apache.calcite.rel.core.{Aggregate, AggregateCall, Calc}
+import org.apache.calcite.rel.core._
import org.apache.calcite.rex._
import org.apache.calcite.sql.`type`.SqlTypeFamily
import org.apache.calcite.sql.SqlKind
@@ -188,7 +188,7 @@ object WindowUtil {
*/
def convertToWindowingStrategy(
windowCall: RexCall,
- inputRowType: RelDataType): TimeAttributeWindowingStrategy = {
+ scanInput: RelNode): TimeAttributeWindowingStrategy = {
if (!isWindowTableFunctionCall(windowCall)) {
throw new IllegalArgumentException(
s"RexCall $windowCall is not a window table-valued " +
@@ -196,6 +196,7 @@ object WindowUtil {
}
val timeIndex = getTimeAttributeIndex(windowCall.operands(0))
+ val inputRowType = scanInput.getRowType
val fieldType = inputRowType.getFieldList.get(timeIndex).getType
val timeAttributeType = FlinkTypeFactory.toLogicalType(fieldType)
if (!canBeTimeAttributeType(timeAttributeType)) {
@@ -234,6 +235,22 @@ object WindowUtil {
val step = getOperandAsLong(windowCall.operands(1))
val maxSize = getOperandAsLong(windowCall.operands(2))
new CumulativeWindowSpec(Duration.ofMillis(maxSize), Duration.ofMillis(step), offset)
+ case FlinkSqlOperatorTable.SESSION =>
+ windowCall match {
+ // with syntax partition key
+ case setSemanticsTableCall: RexSetSemanticsTableCall =>
+ if (!setSemanticsTableCall.getOrderKeys.isEmpty) {
+ throw new ValidationException("Session window TVF doesn't support order by clause.")
+ }
+ val gap = getOperandAsLong(windowCall.operands(1))
+ new SessionWindowSpec(
+ Duration.ofMillis(gap),
+ windowCall.asInstanceOf[RexSetSemanticsTableCall].getPartitionKeys)
+ // without syntax partition key
+ case _ =>
+ val gap = getOperandAsLong(windowCall.operands(1))
+ new SessionWindowSpec(Duration.ofMillis(gap), new Array[Int](0))
+ }
}
new TimeAttributeWindowingStrategy(windowSpec, timeAttributeType, timeIndex)
@@ -317,11 +334,17 @@ object WindowUtil {
/**
* For rowtime window, return true if the given aggregate grouping contains window start and end.
* For proctime window, we should also check if it exists a neighbour windowTableFunctionCall.
+ *
+ * If the window is a session window, we should also check if the partition keys are the same as
+ * the group keys. See more at [[WindowUtil.validGroupKeyPartitionKey()]].
*/
def isValidWindowAggregate(agg: FlinkLogicalAggregate): Boolean = {
val fmq = FlinkRelMetadataQuery.reuseOrCreate(agg.getCluster.getMetadataQuery)
val windowProperties = fmq.getRelWindowProperties(agg.getInput)
val grouping = agg.getGroupSet
+ if (!validGroupKeyPartitionKey(grouping, windowProperties)) {
+ return false
+ }
if (WindowUtil.groupingContainsWindowStartEnd(grouping, windowProperties)) {
windowProperties.isRowtime || existNeighbourWindowTableFunc(agg.getInput)
} else {
@@ -392,4 +415,54 @@ object WindowUtil {
}
false
}
+
+ /**
+ * This method only checks the window like session window that contains partition keys. The
+ * partition keys in session window should be the same as the group keys in aggregate. If they are
+ * different, the downstream will not be able to use window-related optimizations, and the window
+ * table function scan will always be an isolated node.
+ *
+ * Take a SQL following as an example.
+ *
+ * {{{
+ * Source Table `my_table` SCHEMA: a int, b int, c int, proctime as PROCTIME()
+ * SQL: SELECT count(c) FROM
+ * TABLE(SESSION(
+ * TABLE my_table PARTITION BY (a, b),
+ * DESCRIPTOR(proctime),
+ * INTERVAL '5' MINUTE))
+ * GROUP BY a, window_start, window_end
+ * }}}
+ *
+ * The plan is like:
+ * {{{
+ * FlinkLogicalAggregate(group=[{0, 1, 2}], EXPR$0=[COUNT($3)])
+ * FlinkLogicalCalc(select=[b, window_start, window_end, c])
+ * FlinkLogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0, $1),
+ * DESCRIPTOR($3), 300000:INTERVAL MINUTE)]])
+ * FlinkLogicalCalc(select=[a, b, c, PROCTIME() AS proctime])
+ * FlinkLogicalTableSourceScan(table=[[default_catalog, default_database, my_table]],
+ * fields=[a, b, c])
+ * }}}
+ *
+ * In this case, the group keys in Aggregate are different with the partition keys in Session
+ * Window in TableFunctionScan. The Aggregate node should not be converted into WindowAggregate
+ * finally, because the data from source has been split, resolved and aggregated in different
+ * subtasks in FlinkLogicalTableSourceScan with different partition keys.
+ */
+ private def validGroupKeyPartitionKey(
+ grouping: ImmutableBitSet,
+ windowProp: RelWindowProperties): Boolean = {
+ if (windowProp == null) {
+ return false
+ }
+ val (_, _, _, newGrouping) =
+ WindowUtil.groupingExcludeWindowStartEndTimeColumns(grouping, windowProp)
+ windowProp.getWindowSpec match {
+ case session: SessionWindowSpec =>
+ val partitionKeys = session.getPartitionKeyIndices.toSet
+ partitionKeys.equals(newGrouping.toSet)
+ case _ => true
+ }
+ }
}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest.java
index 4aae87aab60..413b5a30062 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest.java
@@ -449,4 +449,80 @@ class WindowAggregateJsonPlanTest extends TableTestBase {
+ " CUMULATE(TABLE MyTable, DESCRIPTOR(rowtime), INTERVAL '10' MINUTE, INTERVAL '1' HOUR)) "
+ "GROUP BY a, window_start, window_end");
}
+
+ @Test
+ void testProcTimeSessionWindowWithoutPartitionKey() {
+ String sinkTableDdl =
+ "CREATE TABLE MySink (\n"
+ + " window_end TIMESTAMP(3),\n"
+ + " cnt BIGINT\n"
+ + ") WITH (\n"
+ + " 'connector' = 'values')\n";
+ tEnv.executeSql(sinkTableDdl);
+ util.verifyJsonPlan(
+ "insert into MySink select\n"
+ + " window_end,\n"
+ + " COUNT(*)\n"
+ + "FROM TABLE(\n"
+ + " SESSION(TABLE MyTable, DESCRIPTOR(proctime), INTERVAL '15' MINUTE))\n"
+ + "GROUP BY window_start, window_end");
+ }
+
+ @Test
+ void testProcTimeSessionWindowWithPartitionKey() {
+ String sinkTableDdl =
+ "CREATE TABLE MySink (\n"
+ + " b BIGINT,\n"
+ + " window_end TIMESTAMP(3),\n"
+ + " cnt BIGINT\n"
+ + ") WITH (\n"
+ + " 'connector' = 'values')\n";
+ tEnv.executeSql(sinkTableDdl);
+ util.verifyJsonPlan(
+ "insert into MySink select\n"
+ + " b,\n"
+ + " window_end,\n"
+ + " COUNT(*)\n"
+ + "FROM TABLE(\n"
+ + " SESSION(TABLE MyTable PARTITION BY b, DESCRIPTOR(proctime), INTERVAL '15' MINUTE))\n"
+ + "GROUP BY b, window_start, window_end");
+ }
+
+ @Test
+ void testEventTimeSessionWindowWithoutPartitionKey() {
+ String sinkTableDdl =
+ "CREATE TABLE MySink (\n"
+ + " window_end TIMESTAMP(3),\n"
+ + " cnt BIGINT\n"
+ + ") WITH (\n"
+ + " 'connector' = 'values')\n";
+ tEnv.executeSql(sinkTableDdl);
+ util.verifyJsonPlan(
+ "insert into MySink select\n"
+ + " window_end,\n"
+ + " COUNT(*)\n"
+ + "FROM TABLE(\n"
+ + " SESSION(TABLE MyTable, DESCRIPTOR(rowtime), INTERVAL '15' MINUTE))\n"
+ + "GROUP BY window_start, window_end");
+ }
+
+ @Test
+ void testEventTimeSessionWindowWithPartitionKey() {
+ String sinkTableDdl =
+ "CREATE TABLE MySink (\n"
+ + " b BIGINT,\n"
+ + " window_end TIMESTAMP(3),\n"
+ + " cnt BIGINT\n"
+ + ") WITH (\n"
+ + " 'connector' = 'values')\n";
+ tEnv.executeSql(sinkTableDdl);
+ util.verifyJsonPlan(
+ "insert into MySink select\n"
+ + " b,\n"
+ + " window_end,\n"
+ + " COUNT(*)\n"
+ + "FROM TABLE(\n"
+ + " SESSION(TABLE MyTable PARTITION BY b, DESCRIPTOR(rowtime), INTERVAL '15' MINUTE))\n"
+ + "GROUP BY b, window_start, window_end");
+ }
}
diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/jsonplan/WindowAggregateJsonITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/jsonplan/WindowAggregateJsonITCase.java
index f636ee8bb55..47983bd3d2e 100644
--- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/jsonplan/WindowAggregateJsonITCase.java
+++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/jsonplan/WindowAggregateJsonITCase.java
@@ -219,4 +219,25 @@ class WindowAggregateJsonITCase extends JsonPlanTestBase {
"+I[null, 7.0, 1]"),
result);
}
+
+ @TestTemplate
+ public void testEventTimeSessionWindow() throws Exception {
+ createTestValuesSinkTable("MySink", "name STRING", "cnt BIGINT");
+ compileSqlAndExecutePlan(
+ "insert into MySink select\n"
+ + " name,\n"
+ + " COUNT(*)\n"
+ + "FROM TABLE(\n"
+ + " SESSION(\n"
+ + " TABLE MyTable PARTITION BY name,\n"
+ + " DESCRIPTOR(rowtime),\n"
+ + " INTERVAL '5' SECOND))"
+ + "GROUP BY name, window_start, window_end")
+ .await();
+
+ List<String> result = TestValuesTableFactory.getResultsAsStrings("MySink");
+ assertResult(
+ Arrays.asList("+I[a, 6]", "+I[b, 1]", "+I[b, 1]", "+I[b, 2]", "+I[null, 1]"),
+ result);
+ }
}
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testEventTimeSessionWindowWithPartitionKey.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testEventTimeSessionWindowWithPartitionKey.out
new file mode 100644
index 00000000000..ac07c37f332
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testEventTimeSessionWindowWithPartitionKey.out
@@ -0,0 +1,373 @@
+{
+ "flinkVersion" : "",
+ "nodes" : [ {
+ "id" : 1,
+ "type" : "stream-exec-table-source-scan_1",
+ "scanTableSource" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`MyTable`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "a",
+ "dataType" : "INT"
+ }, {
+ "name" : "b",
+ "dataType" : "BIGINT"
+ }, {
+ "name" : "c",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "rowtime",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "TO_TIMESTAMP(`c`)"
+ }
+ }, {
+ "name" : "proctime",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$PROCTIME$1",
+ "operands" : [ ],
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ },
+ "serializableString" : "PROCTIME()"
+ }
+ } ],
+ "watermarkSpecs" : [ {
+ "rowtimeAttribute" : "rowtime",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$-$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 3,
+ "type" : "TIMESTAMP(3)"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "1000",
+ "type" : "INTERVAL SECOND(6) NOT NULL"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "`rowtime` - INTERVAL '1' SECOND"
+ }
+ } ]
+ },
+ "partitionKeys" : [ ],
+ "options" : {
+ "connector" : "values"
+ }
+ }
+ },
+ "abilities" : [ {
+ "type" : "ProjectPushDown",
+ "projectedFields" : [ [ 1 ], [ 2 ] ],
+ "producedType" : "ROW<`b` BIGINT, `c` VARCHAR(2147483647)> NOT NULL"
+ }, {
+ "type" : "ReadingMetadata",
+ "metadataKeys" : [ ],
+ "producedType" : "ROW<`b` BIGINT, `c` VARCHAR(2147483647)> NOT NULL"
+ } ]
+ },
+ "outputType" : "ROW<`b` BIGINT, `c` VARCHAR(2147483647)>",
+ "description" : "TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[b, c], metadata=[]]], fields=[b, c])",
+ "inputProperties" : [ ]
+ }, {
+ "id" : 2,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "BIGINT"
+ }, {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 1,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`b` BIGINT, `rowtime` TIMESTAMP(3)>",
+ "description" : "Calc(select=[b, TO_TIMESTAMP(c) AS rowtime])"
+ }, {
+ "id" : 3,
+ "type" : "stream-exec-watermark-assigner_1",
+ "watermarkExpr" : {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$-$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 1,
+ "type" : "TIMESTAMP(3)"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "1000",
+ "type" : "INTERVAL SECOND(6) NOT NULL"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "rowtimeFieldIndex" : 1,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "b",
+ "fieldType" : "BIGINT"
+ }, {
+ "name" : "rowtime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ } ]
+ },
+ "description" : "WatermarkAssigner(rowtime=[rowtime], watermark=[(rowtime - 1000:INTERVAL SECOND)])"
+ }, {
+ "id" : 4,
+ "type" : "stream-exec-exchange_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "HASH",
+ "keys" : [ 0 ]
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "b",
+ "fieldType" : "BIGINT"
+ }, {
+ "name" : "rowtime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ } ]
+ },
+ "description" : "Exchange(distribution=[hash[b]])"
+ }, {
+ "id" : 5,
+ "type" : "stream-exec-window-aggregate_1",
+ "configuration" : {
+ "table.local-time-zone" : "default"
+ },
+ "grouping" : [ 0 ],
+ "aggCalls" : [ {
+ "name" : "EXPR$2",
+ "syntax" : "FUNCTION_STAR",
+ "internalName" : "$COUNT$1",
+ "argList" : [ ],
+ "filterArg" : -1,
+ "distinct" : false,
+ "approximate" : false,
+ "ignoreNulls" : false,
+ "type" : "BIGINT NOT NULL"
+ } ],
+ "windowing" : {
+ "strategy" : "TimeAttribute",
+ "window" : {
+ "type" : "SessionWindow",
+ "gap" : "PT15M",
+ "partition_key_indices" : [ 0 ]
+ },
+ "timeAttributeType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ },
+ "timeAttributeIndex" : 1,
+ "isRowtime" : true
+ },
+ "namedWindowProperties" : [ {
+ "name" : "window_start",
+ "property" : {
+ "kind" : "WindowStart",
+ "reference" : {
+ "name" : "w$",
+ "type" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ }
+ }
+ }, {
+ "name" : "window_end",
+ "property" : {
+ "kind" : "WindowEnd",
+ "reference" : {
+ "name" : "w$",
+ "type" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ }
+ }
+ } ],
+ "needRetraction" : false,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`b` BIGINT, `EXPR$2` BIGINT NOT NULL, `window_start` TIMESTAMP(3) NOT NULL, `window_end` TIMESTAMP(3) NOT NULL>",
+ "description" : "WindowAggregate(groupBy=[b], window=[SESSION(time_col=[rowtime], gap=[15 min], partition keys=[b])], select=[b, COUNT(*) AS EXPR$2, start('w$) AS window_start, end('w$) AS window_end])"
+ }, {
+ "id" : 6,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "BIGINT"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 3,
+ "type" : "TIMESTAMP(3) NOT NULL"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 1,
+ "type" : "BIGINT NOT NULL"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`b` BIGINT, `window_end` TIMESTAMP(3) NOT NULL, `EXPR$2` BIGINT NOT NULL>",
+ "description" : "Calc(select=[b, window_end, EXPR$2])"
+ }, {
+ "id" : 7,
+ "type" : "stream-exec-sink_1",
+ "configuration" : {
+ "table.exec.sink.keyed-shuffle" : "AUTO",
+ "table.exec.sink.not-null-enforcer" : "ERROR",
+ "table.exec.sink.rowtime-inserter" : "ENABLED",
+ "table.exec.sink.type-length-enforcer" : "IGNORE",
+ "table.exec.sink.upsert-materialize" : "AUTO"
+ },
+ "dynamicTableSink" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`MySink`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "b",
+ "dataType" : "BIGINT"
+ }, {
+ "name" : "window_end",
+ "dataType" : "TIMESTAMP(3)"
+ }, {
+ "name" : "cnt",
+ "dataType" : "BIGINT"
+ } ],
+ "watermarkSpecs" : [ ]
+ },
+ "partitionKeys" : [ ],
+ "options" : {
+ "connector" : "values"
+ }
+ }
+ }
+ },
+ "inputChangelogMode" : [ "INSERT" ],
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`b` BIGINT, `window_end` TIMESTAMP(3) NOT NULL, `EXPR$2` BIGINT NOT NULL>",
+ "description" : "Sink(table=[default_catalog.default_database.MySink], fields=[b, window_end, EXPR$2])"
+ } ],
+ "edges" : [ {
+ "source" : 1,
+ "target" : 2,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 2,
+ "target" : 3,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 3,
+ "target" : 4,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 4,
+ "target" : 5,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 5,
+ "target" : 6,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 6,
+ "target" : 7,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ } ]
+}
\ No newline at end of file
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testEventTimeSessionWindowWithoutPartitionKey.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testEventTimeSessionWindowWithoutPartitionKey.out
new file mode 100644
index 00000000000..8fc9b4a3b35
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testEventTimeSessionWindowWithoutPartitionKey.out
@@ -0,0 +1,355 @@
+{
+ "flinkVersion" : "",
+ "nodes" : [ {
+ "id" : 1,
+ "type" : "stream-exec-table-source-scan_1",
+ "scanTableSource" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`MyTable`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "a",
+ "dataType" : "INT"
+ }, {
+ "name" : "b",
+ "dataType" : "BIGINT"
+ }, {
+ "name" : "c",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "rowtime",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "TO_TIMESTAMP(`c`)"
+ }
+ }, {
+ "name" : "proctime",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$PROCTIME$1",
+ "operands" : [ ],
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ },
+ "serializableString" : "PROCTIME()"
+ }
+ } ],
+ "watermarkSpecs" : [ {
+ "rowtimeAttribute" : "rowtime",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$-$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 3,
+ "type" : "TIMESTAMP(3)"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "1000",
+ "type" : "INTERVAL SECOND(6) NOT NULL"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "`rowtime` - INTERVAL '1' SECOND"
+ }
+ } ]
+ },
+ "partitionKeys" : [ ],
+ "options" : {
+ "connector" : "values"
+ }
+ }
+ },
+ "abilities" : [ {
+ "type" : "ProjectPushDown",
+ "projectedFields" : [ [ 2 ] ],
+ "producedType" : "ROW<`c` VARCHAR(2147483647)> NOT NULL"
+ }, {
+ "type" : "ReadingMetadata",
+ "metadataKeys" : [ ],
+ "producedType" : "ROW<`c` VARCHAR(2147483647)> NOT NULL"
+ } ]
+ },
+ "outputType" : "ROW<`c` VARCHAR(2147483647)>",
+ "description" : "TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[c], metadata=[]]], fields=[c])",
+ "inputProperties" : [ ]
+ }, {
+ "id" : 2,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`rowtime` TIMESTAMP(3)>",
+ "description" : "Calc(select=[TO_TIMESTAMP(c) AS rowtime])"
+ }, {
+ "id" : 3,
+ "type" : "stream-exec-watermark-assigner_1",
+ "watermarkExpr" : {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$-$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "TIMESTAMP(3)"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "1000",
+ "type" : "INTERVAL SECOND(6) NOT NULL"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "rowtimeFieldIndex" : 0,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "rowtime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ } ]
+ },
+ "description" : "WatermarkAssigner(rowtime=[rowtime], watermark=[(rowtime - 1000:INTERVAL SECOND)])"
+ }, {
+ "id" : 4,
+ "type" : "stream-exec-exchange_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "SINGLETON"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "rowtime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ } ]
+ },
+ "description" : "Exchange(distribution=[single])"
+ }, {
+ "id" : 5,
+ "type" : "stream-exec-window-aggregate_1",
+ "configuration" : {
+ "table.local-time-zone" : "default"
+ },
+ "grouping" : [ ],
+ "aggCalls" : [ {
+ "name" : "EXPR$1",
+ "syntax" : "FUNCTION_STAR",
+ "internalName" : "$COUNT$1",
+ "argList" : [ ],
+ "filterArg" : -1,
+ "distinct" : false,
+ "approximate" : false,
+ "ignoreNulls" : false,
+ "type" : "BIGINT NOT NULL"
+ } ],
+ "windowing" : {
+ "strategy" : "TimeAttribute",
+ "window" : {
+ "type" : "SessionWindow",
+ "gap" : "PT15M",
+ "partition_key_indices" : [ ]
+ },
+ "timeAttributeType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ },
+ "timeAttributeIndex" : 0,
+ "isRowtime" : true
+ },
+ "namedWindowProperties" : [ {
+ "name" : "window_start",
+ "property" : {
+ "kind" : "WindowStart",
+ "reference" : {
+ "name" : "w$",
+ "type" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ }
+ }
+ }, {
+ "name" : "window_end",
+ "property" : {
+ "kind" : "WindowEnd",
+ "reference" : {
+ "name" : "w$",
+ "type" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ }
+ }
+ } ],
+ "needRetraction" : false,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`EXPR$1` BIGINT NOT NULL, `window_start` TIMESTAMP(3) NOT NULL, `window_end` TIMESTAMP(3) NOT NULL>",
+ "description" : "WindowAggregate(window=[SESSION(time_col=[rowtime], gap=[15 min])], select=[COUNT(*) AS EXPR$1, start('w$) AS window_start, end('w$) AS window_end])"
+ }, {
+ "id" : 6,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "TIMESTAMP(3) NOT NULL"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "BIGINT NOT NULL"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`window_end` TIMESTAMP(3) NOT NULL, `EXPR$1` BIGINT NOT NULL>",
+ "description" : "Calc(select=[window_end, EXPR$1])"
+ }, {
+ "id" : 7,
+ "type" : "stream-exec-sink_1",
+ "configuration" : {
+ "table.exec.sink.keyed-shuffle" : "AUTO",
+ "table.exec.sink.not-null-enforcer" : "ERROR",
+ "table.exec.sink.rowtime-inserter" : "ENABLED",
+ "table.exec.sink.type-length-enforcer" : "IGNORE",
+ "table.exec.sink.upsert-materialize" : "AUTO"
+ },
+ "dynamicTableSink" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`MySink`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "window_end",
+ "dataType" : "TIMESTAMP(3)"
+ }, {
+ "name" : "cnt",
+ "dataType" : "BIGINT"
+ } ],
+ "watermarkSpecs" : [ ]
+ },
+ "partitionKeys" : [ ],
+ "options" : {
+ "connector" : "values"
+ }
+ }
+ }
+ },
+ "inputChangelogMode" : [ "INSERT" ],
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`window_end` TIMESTAMP(3) NOT NULL, `EXPR$1` BIGINT NOT NULL>",
+ "description" : "Sink(table=[default_catalog.default_database.MySink], fields=[window_end, EXPR$1])"
+ } ],
+ "edges" : [ {
+ "source" : 1,
+ "target" : 2,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 2,
+ "target" : 3,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 3,
+ "target" : 4,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 4,
+ "target" : 5,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 5,
+ "target" : 6,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 6,
+ "target" : 7,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ } ]
+}
\ No newline at end of file
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testProcTimeSessionWindowWithPartitionKey.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testProcTimeSessionWindowWithPartitionKey.out
new file mode 100644
index 00000000000..059542e81eb
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testProcTimeSessionWindowWithPartitionKey.out
@@ -0,0 +1,460 @@
+{
+ "flinkVersion" : "",
+ "nodes" : [ {
+ "id" : 1,
+ "type" : "stream-exec-table-source-scan_1",
+ "scanTableSource" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`MyTable`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "a",
+ "dataType" : "INT"
+ }, {
+ "name" : "b",
+ "dataType" : "BIGINT"
+ }, {
+ "name" : "c",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "rowtime",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "TO_TIMESTAMP(`c`)"
+ }
+ }, {
+ "name" : "proctime",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$PROCTIME$1",
+ "operands" : [ ],
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ },
+ "serializableString" : "PROCTIME()"
+ }
+ } ],
+ "watermarkSpecs" : [ {
+ "rowtimeAttribute" : "rowtime",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$-$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 3,
+ "type" : "TIMESTAMP(3)"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "1000",
+ "type" : "INTERVAL SECOND(6) NOT NULL"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "`rowtime` - INTERVAL '1' SECOND"
+ }
+ } ]
+ },
+ "partitionKeys" : [ ],
+ "options" : {
+ "connector" : "values"
+ }
+ }
+ },
+ "abilities" : [ {
+ "type" : "ProjectPushDown",
+ "projectedFields" : [ [ 1 ], [ 2 ] ],
+ "producedType" : "ROW<`b` BIGINT, `c` VARCHAR(2147483647)> NOT NULL"
+ }, {
+ "type" : "ReadingMetadata",
+ "metadataKeys" : [ ],
+ "producedType" : "ROW<`b` BIGINT, `c` VARCHAR(2147483647)> NOT NULL"
+ } ]
+ },
+ "outputType" : "ROW<`b` BIGINT, `c` VARCHAR(2147483647)>",
+ "description" : "TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[b, c], metadata=[]]], fields=[b, c])",
+ "inputProperties" : [ ]
+ }, {
+ "id" : 2,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "BIGINT"
+ }, {
+ "kind" : "CALL",
+ "internalName" : "$PROCTIME$1",
+ "operands" : [ ],
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }, {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 1,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "b",
+ "fieldType" : "BIGINT"
+ }, {
+ "name" : "proctime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }, {
+ "name" : "rowtime",
+ "fieldType" : "TIMESTAMP(3)"
+ } ]
+ },
+ "description" : "Calc(select=[b, PROCTIME() AS proctime, TO_TIMESTAMP(c) AS rowtime])"
+ }, {
+ "id" : 3,
+ "type" : "stream-exec-watermark-assigner_1",
+ "watermarkExpr" : {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$-$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "TIMESTAMP(3)"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "1000",
+ "type" : "INTERVAL SECOND(6) NOT NULL"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "rowtimeFieldIndex" : 2,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "b",
+ "fieldType" : "BIGINT"
+ }, {
+ "name" : "proctime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }, {
+ "name" : "rowtime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ } ]
+ },
+ "description" : "WatermarkAssigner(rowtime=[rowtime], watermark=[(rowtime - 1000:INTERVAL SECOND)])"
+ }, {
+ "id" : 4,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "BIGINT"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 1,
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "b",
+ "fieldType" : "BIGINT"
+ }, {
+ "name" : "proctime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ } ]
+ },
+ "description" : "Calc(select=[b, proctime])"
+ }, {
+ "id" : 5,
+ "type" : "stream-exec-exchange_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "HASH",
+ "keys" : [ 0 ]
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "b",
+ "fieldType" : "BIGINT"
+ }, {
+ "name" : "proctime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ } ]
+ },
+ "description" : "Exchange(distribution=[hash[b]])"
+ }, {
+ "id" : 6,
+ "type" : "stream-exec-window-aggregate_1",
+ "configuration" : {
+ "table.local-time-zone" : "default"
+ },
+ "grouping" : [ 0 ],
+ "aggCalls" : [ {
+ "name" : "EXPR$2",
+ "syntax" : "FUNCTION_STAR",
+ "internalName" : "$COUNT$1",
+ "argList" : [ ],
+ "filterArg" : -1,
+ "distinct" : false,
+ "approximate" : false,
+ "ignoreNulls" : false,
+ "type" : "BIGINT NOT NULL"
+ } ],
+ "windowing" : {
+ "strategy" : "TimeAttribute",
+ "window" : {
+ "type" : "SessionWindow",
+ "gap" : "PT15M",
+ "partition_key_indices" : [ 0 ]
+ },
+ "timeAttributeType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ },
+ "timeAttributeIndex" : 1,
+ "isRowtime" : false
+ },
+ "namedWindowProperties" : [ {
+ "name" : "window_start",
+ "property" : {
+ "kind" : "WindowStart",
+ "reference" : {
+ "name" : "w$",
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }
+ }
+ }, {
+ "name" : "window_end",
+ "property" : {
+ "kind" : "WindowEnd",
+ "reference" : {
+ "name" : "w$",
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }
+ }
+ } ],
+ "needRetraction" : false,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`b` BIGINT, `EXPR$2` BIGINT NOT NULL, `window_start` TIMESTAMP(3) NOT NULL, `window_end` TIMESTAMP(3) NOT NULL>",
+ "description" : "WindowAggregate(groupBy=[b], window=[SESSION(time_col=[proctime], gap=[15 min], partition keys=[b])], select=[b, COUNT(*) AS EXPR$2, start('w$) AS window_start, end('w$) AS window_end])"
+ }, {
+ "id" : 7,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "BIGINT"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 3,
+ "type" : "TIMESTAMP(3) NOT NULL"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 1,
+ "type" : "BIGINT NOT NULL"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`b` BIGINT, `window_end` TIMESTAMP(3) NOT NULL, `EXPR$2` BIGINT NOT NULL>",
+ "description" : "Calc(select=[b, window_end, EXPR$2])"
+ }, {
+ "id" : 8,
+ "type" : "stream-exec-sink_1",
+ "configuration" : {
+ "table.exec.sink.keyed-shuffle" : "AUTO",
+ "table.exec.sink.not-null-enforcer" : "ERROR",
+ "table.exec.sink.rowtime-inserter" : "ENABLED",
+ "table.exec.sink.type-length-enforcer" : "IGNORE",
+ "table.exec.sink.upsert-materialize" : "AUTO"
+ },
+ "dynamicTableSink" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`MySink`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "b",
+ "dataType" : "BIGINT"
+ }, {
+ "name" : "window_end",
+ "dataType" : "TIMESTAMP(3)"
+ }, {
+ "name" : "cnt",
+ "dataType" : "BIGINT"
+ } ],
+ "watermarkSpecs" : [ ]
+ },
+ "partitionKeys" : [ ],
+ "options" : {
+ "connector" : "values"
+ }
+ }
+ }
+ },
+ "inputChangelogMode" : [ "INSERT" ],
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`b` BIGINT, `window_end` TIMESTAMP(3) NOT NULL, `EXPR$2` BIGINT NOT NULL>",
+ "description" : "Sink(table=[default_catalog.default_database.MySink], fields=[b, window_end, EXPR$2])"
+ } ],
+ "edges" : [ {
+ "source" : 1,
+ "target" : 2,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 2,
+ "target" : 3,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 3,
+ "target" : 4,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 4,
+ "target" : 5,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 5,
+ "target" : 6,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 6,
+ "target" : 7,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 7,
+ "target" : 8,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ } ]
+}
\ No newline at end of file
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testProcTimeSessionWindowWithoutPartitionKey.out b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testProcTimeSessionWindowWithoutPartitionKey.out
new file mode 100644
index 00000000000..d493e0c3f3a
--- /dev/null
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/nodes/exec/stream/WindowAggregateJsonPlanTest_jsonplan/testProcTimeSessionWindowWithoutPartitionKey.out
@@ -0,0 +1,432 @@
+{
+ "flinkVersion" : "",
+ "nodes" : [ {
+ "id" : 1,
+ "type" : "stream-exec-table-source-scan_1",
+ "scanTableSource" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`MyTable`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "a",
+ "dataType" : "INT"
+ }, {
+ "name" : "b",
+ "dataType" : "BIGINT"
+ }, {
+ "name" : "c",
+ "dataType" : "VARCHAR(2147483647)"
+ }, {
+ "name" : "rowtime",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "TO_TIMESTAMP(`c`)"
+ }
+ }, {
+ "name" : "proctime",
+ "kind" : "COMPUTED",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "internalName" : "$PROCTIME$1",
+ "operands" : [ ],
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ },
+ "serializableString" : "PROCTIME()"
+ }
+ } ],
+ "watermarkSpecs" : [ {
+ "rowtimeAttribute" : "rowtime",
+ "expression" : {
+ "rexNode" : {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$-$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 3,
+ "type" : "TIMESTAMP(3)"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "1000",
+ "type" : "INTERVAL SECOND(6) NOT NULL"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "serializableString" : "`rowtime` - INTERVAL '1' SECOND"
+ }
+ } ]
+ },
+ "partitionKeys" : [ ],
+ "options" : {
+ "connector" : "values"
+ }
+ }
+ },
+ "abilities" : [ {
+ "type" : "ProjectPushDown",
+ "projectedFields" : [ [ 2 ] ],
+ "producedType" : "ROW<`c` VARCHAR(2147483647)> NOT NULL"
+ }, {
+ "type" : "ReadingMetadata",
+ "metadataKeys" : [ ],
+ "producedType" : "ROW<`c` VARCHAR(2147483647)> NOT NULL"
+ } ]
+ },
+ "outputType" : "ROW<`c` VARCHAR(2147483647)>",
+ "description" : "TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[c], metadata=[]]], fields=[c])",
+ "inputProperties" : [ ]
+ }, {
+ "id" : 2,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "CALL",
+ "internalName" : "$PROCTIME$1",
+ "operands" : [ ],
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }, {
+ "kind" : "CALL",
+ "internalName" : "$TO_TIMESTAMP$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "VARCHAR(2147483647)"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "proctime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }, {
+ "name" : "rowtime",
+ "fieldType" : "TIMESTAMP(3)"
+ } ]
+ },
+ "description" : "Calc(select=[PROCTIME() AS proctime, TO_TIMESTAMP(c) AS rowtime])"
+ }, {
+ "id" : 3,
+ "type" : "stream-exec-watermark-assigner_1",
+ "watermarkExpr" : {
+ "kind" : "CALL",
+ "syntax" : "SPECIAL",
+ "internalName" : "$-$1",
+ "operands" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 1,
+ "type" : "TIMESTAMP(3)"
+ }, {
+ "kind" : "LITERAL",
+ "value" : "1000",
+ "type" : "INTERVAL SECOND(6) NOT NULL"
+ } ],
+ "type" : "TIMESTAMP(3)"
+ },
+ "rowtimeFieldIndex" : 1,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "proctime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }, {
+ "name" : "rowtime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITHOUT_TIME_ZONE",
+ "precision" : 3,
+ "kind" : "ROWTIME"
+ }
+ } ]
+ },
+ "description" : "WatermarkAssigner(rowtime=[rowtime], watermark=[(rowtime - 1000:INTERVAL SECOND)])"
+ }, {
+ "id" : 4,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "proctime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ } ]
+ },
+ "description" : "Calc(select=[proctime])"
+ }, {
+ "id" : 5,
+ "type" : "stream-exec-exchange_1",
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "SINGLETON"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "name" : "proctime",
+ "fieldType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ } ]
+ },
+ "description" : "Exchange(distribution=[single])"
+ }, {
+ "id" : 6,
+ "type" : "stream-exec-window-aggregate_1",
+ "configuration" : {
+ "table.local-time-zone" : "default"
+ },
+ "grouping" : [ ],
+ "aggCalls" : [ {
+ "name" : "EXPR$1",
+ "syntax" : "FUNCTION_STAR",
+ "internalName" : "$COUNT$1",
+ "argList" : [ ],
+ "filterArg" : -1,
+ "distinct" : false,
+ "approximate" : false,
+ "ignoreNulls" : false,
+ "type" : "BIGINT NOT NULL"
+ } ],
+ "windowing" : {
+ "strategy" : "TimeAttribute",
+ "window" : {
+ "type" : "SessionWindow",
+ "gap" : "PT15M",
+ "partition_key_indices" : [ ]
+ },
+ "timeAttributeType" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ },
+ "timeAttributeIndex" : 0,
+ "isRowtime" : false
+ },
+ "namedWindowProperties" : [ {
+ "name" : "window_start",
+ "property" : {
+ "kind" : "WindowStart",
+ "reference" : {
+ "name" : "w$",
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }
+ }
+ }, {
+ "name" : "window_end",
+ "property" : {
+ "kind" : "WindowEnd",
+ "reference" : {
+ "name" : "w$",
+ "type" : {
+ "type" : "TIMESTAMP_WITH_LOCAL_TIME_ZONE",
+ "nullable" : false,
+ "precision" : 3,
+ "kind" : "PROCTIME"
+ }
+ }
+ }
+ } ],
+ "needRetraction" : false,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`EXPR$1` BIGINT NOT NULL, `window_start` TIMESTAMP(3) NOT NULL, `window_end` TIMESTAMP(3) NOT NULL>",
+ "description" : "WindowAggregate(window=[SESSION(time_col=[proctime], gap=[15 min])], select=[COUNT(*) AS EXPR$1, start('w$) AS window_start, end('w$) AS window_end])"
+ }, {
+ "id" : 7,
+ "type" : "stream-exec-calc_1",
+ "projection" : [ {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 2,
+ "type" : "TIMESTAMP(3) NOT NULL"
+ }, {
+ "kind" : "INPUT_REF",
+ "inputIndex" : 0,
+ "type" : "BIGINT NOT NULL"
+ } ],
+ "condition" : null,
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`window_end` TIMESTAMP(3) NOT NULL, `EXPR$1` BIGINT NOT NULL>",
+ "description" : "Calc(select=[window_end, EXPR$1])"
+ }, {
+ "id" : 8,
+ "type" : "stream-exec-sink_1",
+ "configuration" : {
+ "table.exec.sink.keyed-shuffle" : "AUTO",
+ "table.exec.sink.not-null-enforcer" : "ERROR",
+ "table.exec.sink.rowtime-inserter" : "ENABLED",
+ "table.exec.sink.type-length-enforcer" : "IGNORE",
+ "table.exec.sink.upsert-materialize" : "AUTO"
+ },
+ "dynamicTableSink" : {
+ "table" : {
+ "identifier" : "`default_catalog`.`default_database`.`MySink`",
+ "resolvedTable" : {
+ "schema" : {
+ "columns" : [ {
+ "name" : "window_end",
+ "dataType" : "TIMESTAMP(3)"
+ }, {
+ "name" : "cnt",
+ "dataType" : "BIGINT"
+ } ],
+ "watermarkSpecs" : [ ]
+ },
+ "partitionKeys" : [ ],
+ "options" : {
+ "connector" : "values"
+ }
+ }
+ }
+ },
+ "inputChangelogMode" : [ "INSERT" ],
+ "inputProperties" : [ {
+ "requiredDistribution" : {
+ "type" : "UNKNOWN"
+ },
+ "damBehavior" : "PIPELINED",
+ "priority" : 0
+ } ],
+ "outputType" : "ROW<`window_end` TIMESTAMP(3) NOT NULL, `EXPR$1` BIGINT NOT NULL>",
+ "description" : "Sink(table=[default_catalog.default_database.MySink], fields=[window_end, EXPR$1])"
+ } ],
+ "edges" : [ {
+ "source" : 1,
+ "target" : 2,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 2,
+ "target" : 3,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 3,
+ "target" : 4,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 4,
+ "target" : 5,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 5,
+ "target" : 6,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 6,
+ "target" : 7,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ }, {
+ "source" : 7,
+ "target" : 8,
+ "shuffle" : {
+ "type" : "FORWARD"
+ },
+ "shuffleMode" : "PIPELINED"
+ } ]
+}
\ No newline at end of file
diff --git a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.xml b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.xml
index 002c43c0b31..b5d7fa725cf 100644
--- a/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.xml
+++ b/flink-table/flink-table-planner/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.xml
@@ -1186,6 +1186,70 @@ Calc(select=[window_time, start_time, end_time])
+- Calc(select=[CAST(rowtime AS TIMESTAMP(3)) AS rowtime, rowtime AS rowtime_0])
+- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[rowtime], metadata=[]]], fields=[rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testGroupKeysIndicesChangesInSessionWindow[aggPhaseEnforcer=ONE_PHASE]">
+ <Resource name="explain">
+ <![CDATA[== Abstract Syntax Tree ==
+LogicalProject(window_start=[$0], window_end=[$1], EXPR$2=[$3], a=[$2])
++- LogicalAggregate(group=[{0, 1, 2}], EXPR$2=[COUNT()])
+ +- LogicalProject(window_start=[$7], window_end=[$8], a=[$0])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($6), 600000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP_LTZ(3) *PROCTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+
+== Optimized Physical Plan ==
+Calc(select=[window_start, window_end, EXPR$2, a])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[proctime], gap=[10 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$2, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, proctime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- Calc(select=[a, PROCTIME() AS proctime, rowtime])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, rowtime], metadata=[]]], fields=[a, rowtime])
+
+== Optimized Execution Plan ==
+Calc(select=[window_start, window_end, EXPR$2, a])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[proctime], gap=[10 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$2, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, proctime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[(rowtime - 1000:INTERVAL SECOND)])
+ +- Calc(select=[a, PROCTIME() AS proctime, rowtime])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, rowtime], metadata=[]]], fields=[a, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testGroupKeysIndicesChangesInSessionWindow[aggPhaseEnforcer=TWO_PHASE]">
+ <Resource name="explain">
+ <![CDATA[== Abstract Syntax Tree ==
+LogicalProject(window_start=[$0], window_end=[$1], EXPR$2=[$3], a=[$2])
++- LogicalAggregate(group=[{0, 1, 2}], EXPR$2=[COUNT()])
+ +- LogicalProject(window_start=[$7], window_end=[$8], a=[$0])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($6), 600000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP_LTZ(3) *PROCTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+
+== Optimized Physical Plan ==
+Calc(select=[window_start, window_end, EXPR$2, a])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[proctime], gap=[10 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$2, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, proctime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- Calc(select=[a, PROCTIME() AS proctime, rowtime])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, rowtime], metadata=[]]], fields=[a, rowtime])
+
+== Optimized Execution Plan ==
+Calc(select=[window_start, window_end, EXPR$2, a])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[proctime], gap=[10 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$2, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, proctime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[(rowtime - 1000:INTERVAL SECOND)])
+ +- Calc(select=[a, PROCTIME() AS proctime, rowtime])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, rowtime], metadata=[]]], fields=[a, rowtime])
]]>
</Resource>
</TestCase>
@@ -2170,6 +2234,446 @@ Sink(table=[default_catalog.default_database.sink], fields=[ws, we, b, c])
+- Exchange(distribution=[hash[b]])
+- Calc(select=[b, CASE((a = 11), 1, null:INTEGER) AS $f3, PROCTIME() AS proctime], where=[SEARCH(a, Sarg[1, 5, 7, 9, 11])])
+- TableSourceScan(table=[[default_catalog, default_database, source, project=[a, b], metadata=[]]], fields=[a, b])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSession_DistinctSplitEnabled[aggPhaseEnforcer=ONE_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ a,
+ window_start,
+ window_end,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ count(distinct c) AS uv
+FROM TABLE(
+ SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(proctime), INTERVAL '5' MINUTE))
+GROUP BY a, window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1, 2}], EXPR$3=[COUNT()], EXPR$4=[SUM($3)], EXPR$5=[MAX($3) FILTER $4], uv=[COUNT(DISTINCT $5)])
++- LogicalProject(a=[$0], window_start=[$7], window_end=[$8], d=[$3], $f4=[IS TRUE(>($1, 1000))], c=[$2])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($6), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP_LTZ(3) *PROCTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[a, window_start, window_end, EXPR$3, EXPR$4, EXPR$5, uv])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[proctime], gap=[5 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$3, SUM(d) AS EXPR$4, MAX(d) FILTER $f4 AS EXPR$5, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, d, IS TRUE(>(b, 1000)) AS $f4, c, proctime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- Calc(select=[a, b, c, d, PROCTIME() AS proctime, rowtime])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, b, c, d, rowtime], metadata=[]]], fields=[a, b, c, d, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSession_DistinctSplitEnabled[aggPhaseEnforcer=TWO_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ a,
+ window_start,
+ window_end,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ count(distinct c) AS uv
+FROM TABLE(
+ SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(proctime), INTERVAL '5' MINUTE))
+GROUP BY a, window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1, 2}], EXPR$3=[COUNT()], EXPR$4=[SUM($3)], EXPR$5=[MAX($3) FILTER $4], uv=[COUNT(DISTINCT $5)])
++- LogicalProject(a=[$0], window_start=[$7], window_end=[$8], d=[$3], $f4=[IS TRUE(>($1, 1000))], c=[$2])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($6), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP_LTZ(3) *PROCTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[a, window_start, window_end, EXPR$3, EXPR$4, EXPR$5, uv])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[proctime], gap=[5 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$3, SUM(d) AS EXPR$4, MAX(d) FILTER $f4 AS EXPR$5, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, d, IS TRUE(>(b, 1000)) AS $f4, c, proctime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- Calc(select=[a, b, c, d, PROCTIME() AS proctime, rowtime])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[a, b, c, d, rowtime], metadata=[]]], fields=[a, b, c, d, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSession_OnProctime[aggPhaseEnforcer=TWO_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ a,
+ window_start,
+ window_end,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ weightedAvg(b, e) AS wAvg,
+ count(distinct c) AS uv
+FROM TABLE(
+ SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(proctime), INTERVAL '5' MINUTE))
+GROUP BY a, window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1, 2}], EXPR$3=[COUNT()], EXPR$4=[SUM($3)], EXPR$5=[MAX($3) FILTER $4], wAvg=[weightedAvg($5, $6)], uv=[COUNT(DISTINCT $7)])
++- LogicalProject(a=[$0], window_start=[$7], window_end=[$8], d=[$3], $f4=[IS TRUE(>($1, 1000))], b=[$1], e=[$4], c=[$2])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($6), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP_LTZ(3) *PROCTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[a, window_start, window_end, EXPR$3, EXPR$4, EXPR$5, wAvg, uv])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[proctime], gap=[5 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$3, SUM(d) AS EXPR$4, MAX(d) FILTER $f4 AS EXPR$5, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, d, IS TRUE(>(b, 1000)) AS $f4, b, e, c, proctime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- Calc(select=[a, b, c, d, e, rowtime, PROCTIME() AS proctime])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSession_OnRowtime[aggPhaseEnforcer=ONE_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ a,
+ window_start,
+ window_end,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ weightedAvg(b, e) AS wAvg,
+ count(distinct c) AS uv
+FROM TABLE(
+ SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+GROUP BY a, window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1, 2}], EXPR$3=[COUNT()], EXPR$4=[SUM($3)], EXPR$5=[MAX($3) FILTER $4], wAvg=[weightedAvg($5, $6)], uv=[COUNT(DISTINCT $7)])
++- LogicalProject(a=[$0], window_start=[$7], window_end=[$8], d=[$3], $f4=[IS TRUE(>($1, 1000))], b=[$1], e=[$4], c=[$2])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($5), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP(3) *ROWTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[a, window_start, window_end, EXPR$3, EXPR$4, EXPR$5, wAvg, uv])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[rowtime], gap=[5 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$3, SUM(d) AS EXPR$4, MAX(d) FILTER $f4 AS EXPR$5, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, d, IS TRUE(>(b, 1000)) AS $f4, b, e, c, rowtime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSession_OnRowtime[aggPhaseEnforcer=TWO_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ a,
+ window_start,
+ window_end,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ weightedAvg(b, e) AS wAvg,
+ count(distinct c) AS uv
+FROM TABLE(
+ SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+GROUP BY a, window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1, 2}], EXPR$3=[COUNT()], EXPR$4=[SUM($3)], EXPR$5=[MAX($3) FILTER $4], wAvg=[weightedAvg($5, $6)], uv=[COUNT(DISTINCT $7)])
++- LogicalProject(a=[$0], window_start=[$7], window_end=[$8], d=[$3], $f4=[IS TRUE(>($1, 1000))], b=[$1], e=[$4], c=[$2])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($5), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP(3) *ROWTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[a, window_start, window_end, EXPR$3, EXPR$4, EXPR$5, wAvg, uv])
++- WindowAggregate(groupBy=[a], window=[SESSION(time_col=[rowtime], gap=[5 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$3, SUM(d) AS EXPR$4, MAX(d) FILTER $f4 AS EXPR$5, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, d, IS TRUE(>(b, 1000)) AS $f4, b, e, c, rowtime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSessionWindowTVFWithoutPartitionKeyWhenCantMerge[aggPhaseEnforcer=ONE_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ window_start,
+ window_end,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ weightedAvg(b, e) AS wAvg,
+ count(distinct c) AS uv
+FROM (
+ SELECT window_start, rowtime, d, proctime, e, b, c, window_end, window_time, a
+ FROM TABLE(SESSION(TABLE MyTable, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ WHERE window_start >= TIMESTAMP '2021-01-01 10:10:00.000'
+)
+GROUP BY window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1}], EXPR$2=[COUNT()], EXPR$3=[SUM($2)], EXPR$4=[MAX($2) FILTER $3], wAvg=[weightedAvg($4, $5)], uv=[COUNT(DISTINCT $6)])
++- LogicalProject(window_start=[$0], window_end=[$7], d=[$2], $f3=[IS TRUE(>($5, 1000))], b=[$5], e=[$4], c=[$6])
+ +- LogicalProject(window_start=[$7], rowtime=[$5], d=[$3], proctime=[$6], e=[$4], b=[$1], c=[$2], window_end=[$8], window_time=[$9], a=[$0])
+ +- LogicalFilter(condition=[>=($7, 2021-01-01 10:10:00)])
+ +- LogicalTableFunctionScan(invocation=[SESSION(DESCRIPTOR($5), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP(3) *ROWTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[window_start, window_end, EXPR$2, EXPR$3, EXPR$4, wAvg, uv])
++- WindowAggregate(window=[SESSION(win_start=[window_start], win_end=[window_end], gap=[5 min])], select=[COUNT(*) AS EXPR$2, SUM(d) AS EXPR$3, MAX(d) FILTER $f3 AS EXPR$4, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[single])
+ +- Calc(select=[window_start, window_end, d, IS TRUE(>(b, 1000)) AS $f3, b, e, c], where=[>=(window_start, 2021-01-01 10:10:00)])
+ +- WindowTableFunction(window=[SESSION(time_col=[rowtime], gap=[5 min])])
+ +- Exchange(distribution=[single])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[b, c, d, e, rowtime], metadata=[]]], fields=[b, c, d, e, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSessionWindowTVFWithoutPartitionKeyWhenCantMerge[aggPhaseEnforcer=TWO_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ window_start,
+ window_end,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ weightedAvg(b, e) AS wAvg,
+ count(distinct c) AS uv
+FROM (
+ SELECT window_start, rowtime, d, proctime, e, b, c, window_end, window_time, a
+ FROM TABLE(SESSION(TABLE MyTable, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ WHERE window_start >= TIMESTAMP '2021-01-01 10:10:00.000'
+)
+GROUP BY window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1}], EXPR$2=[COUNT()], EXPR$3=[SUM($2)], EXPR$4=[MAX($2) FILTER $3], wAvg=[weightedAvg($4, $5)], uv=[COUNT(DISTINCT $6)])
++- LogicalProject(window_start=[$0], window_end=[$7], d=[$2], $f3=[IS TRUE(>($5, 1000))], b=[$5], e=[$4], c=[$6])
+ +- LogicalProject(window_start=[$7], rowtime=[$5], d=[$3], proctime=[$6], e=[$4], b=[$1], c=[$2], window_end=[$8], window_time=[$9], a=[$0])
+ +- LogicalFilter(condition=[>=($7, 2021-01-01 10:10:00)])
+ +- LogicalTableFunctionScan(invocation=[SESSION(DESCRIPTOR($5), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP(3) *ROWTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[window_start, window_end, EXPR$2, EXPR$3, EXPR$4, wAvg, uv])
++- WindowAggregate(window=[SESSION(win_start=[window_start], win_end=[window_end], gap=[5 min])], select=[COUNT(*) AS EXPR$2, SUM(d) AS EXPR$3, MAX(d) FILTER $f3 AS EXPR$4, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[single])
+ +- Calc(select=[window_start, window_end, d, IS TRUE(>(b, 1000)) AS $f3, b, e, c], where=[>=(window_start, 2021-01-01 10:10:00)])
+ +- WindowTableFunction(window=[SESSION(time_col=[rowtime], gap=[5 min])])
+ +- Exchange(distribution=[single])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable, project=[b, c, d, e, rowtime], metadata=[]]], fields=[b, c, d, e, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSessionWindowTVFWithPartitionKeyWhenCantMerge[aggPhaseEnforcer=ONE_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ window_start,
+ window_end,
+ a,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ weightedAvg(b, e) AS wAvg,
+ count(distinct c) AS uv
+FROM (
+ SELECT window_start, rowtime, d, proctime, e, b, c, window_end, window_time, a
+ FROM TABLE(SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ WHERE window_start >= TIMESTAMP '2021-01-01 10:10:00.000'
+)
+GROUP BY a, window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(window_start=[$1], window_end=[$2], a=[$0], EXPR$3=[$3], EXPR$4=[$4], EXPR$5=[$5], wAvg=[$6], uv=[$7])
++- LogicalAggregate(group=[{0, 1, 2}], EXPR$3=[COUNT()], EXPR$4=[SUM($3)], EXPR$5=[MAX($3) FILTER $4], wAvg=[weightedAvg($5, $6)], uv=[COUNT(DISTINCT $7)])
+ +- LogicalProject(a=[$9], window_start=[$0], window_end=[$7], d=[$2], $f4=[IS TRUE(>($5, 1000))], b=[$5], e=[$4], c=[$6])
+ +- LogicalProject(window_start=[$7], rowtime=[$5], d=[$3], proctime=[$6], e=[$4], b=[$1], c=[$2], window_end=[$8], window_time=[$9], a=[$0])
+ +- LogicalFilter(condition=[>=($7, 2021-01-01 10:10:00)])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($5), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP(3) *ROWTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[window_start, window_end, a, EXPR$3, EXPR$4, EXPR$5, wAvg, uv])
++- WindowAggregate(groupBy=[a], window=[SESSION(win_start=[window_start], win_end=[window_end], gap=[5 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$3, SUM(d) AS EXPR$4, MAX(d) FILTER $f4 AS EXPR$5, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, window_start, window_end, d, IS TRUE(>(b, 1000)) AS $f4, b, e, c], where=[>=(window_start, 2021-01-01 10:10:00)])
+ +- WindowTableFunction(window=[SESSION(time_col=[rowtime], gap=[5 min], partition keys=[a])])
+ +- Exchange(distribution=[hash[a]])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSessionWindowTVFWithPartitionKeyWhenCantMerge[aggPhaseEnforcer=TWO_PHASE]">
+ <Resource name="sql">
+ <![CDATA[
+SELECT
+ window_start,
+ window_end,
+ a,
+ count(*),
+ sum(d),
+ max(d) filter (where b > 1000),
+ weightedAvg(b, e) AS wAvg,
+ count(distinct c) AS uv
+FROM (
+ SELECT window_start, rowtime, d, proctime, e, b, c, window_end, window_time, a
+ FROM TABLE(SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ WHERE window_start >= TIMESTAMP '2021-01-01 10:10:00.000'
+)
+GROUP BY a, window_start, window_end
+ ]]>
+ </Resource>
+ <Resource name="ast">
+ <![CDATA[
+LogicalProject(window_start=[$1], window_end=[$2], a=[$0], EXPR$3=[$3], EXPR$4=[$4], EXPR$5=[$5], wAvg=[$6], uv=[$7])
++- LogicalAggregate(group=[{0, 1, 2}], EXPR$3=[COUNT()], EXPR$4=[SUM($3)], EXPR$5=[MAX($3) FILTER $4], wAvg=[weightedAvg($5, $6)], uv=[COUNT(DISTINCT $7)])
+ +- LogicalProject(a=[$9], window_start=[$0], window_end=[$7], d=[$2], $f4=[IS TRUE(>($5, 1000))], b=[$5], e=[$4], c=[$6])
+ +- LogicalProject(window_start=[$7], rowtime=[$5], d=[$3], proctime=[$6], e=[$4], b=[$1], c=[$2], window_end=[$8], window_time=[$9], a=[$0])
+ +- LogicalFilter(condition=[>=($7, 2021-01-01 10:10:00)])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($0), DESCRIPTOR($5), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP(3) *ROWTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+]]>
+ </Resource>
+ <Resource name="optimized rel plan">
+ <![CDATA[
+Calc(select=[window_start, window_end, a, EXPR$3, EXPR$4, EXPR$5, wAvg, uv])
++- WindowAggregate(groupBy=[a], window=[SESSION(win_start=[window_start], win_end=[window_end], gap=[5 min], partition keys=[a])], select=[a, COUNT(*) AS EXPR$3, SUM(d) AS EXPR$4, MAX(d) FILTER $f4 AS EXPR$5, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[a]])
+ +- Calc(select=[a, window_start, window_end, d, IS TRUE(>(b, 1000)) AS $f4, b, e, c], where=[>=(window_start, 2021-01-01 10:10:00)])
+ +- WindowTableFunction(window=[SESSION(time_col=[rowtime], gap=[5 min], partition keys=[a])])
+ +- Exchange(distribution=[hash[a]])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSessionWindowWithTwoPartitionKeys[aggPhaseEnforcer=ONE_PHASE]">
+ <Resource name="explain">
+ <![CDATA[== Abstract Syntax Tree ==
+LogicalProject(a=[$1], b=[$0], window_start=[$2], window_end=[$3], EXPR$4=[$4], EXPR$5=[$5], EXPR$6=[$6], wAvg=[$7], uv=[$8])
++- LogicalAggregate(group=[{0, 1, 2, 3}], EXPR$4=[COUNT()], EXPR$5=[SUM($4)], EXPR$6=[MAX($4) FILTER $5], wAvg=[weightedAvg($0, $6)], uv=[COUNT(DISTINCT $7)])
+ +- LogicalProject(b=[$1], a=[$0], window_start=[$7], window_end=[$8], d=[$3], $f5=[IS TRUE(>($1, 1000))], e=[$4], c=[$2])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($1, $0), DESCRIPTOR($5), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP(3) *ROWTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+
+== Optimized Physical Plan ==
+Calc(select=[a, b, window_start, window_end, EXPR$4, EXPR$5, EXPR$6, wAvg, uv])
++- WindowAggregate(groupBy=[b, a], window=[SESSION(time_col=[rowtime], gap=[5 min], partition keys=[b, a])], select=[b, a, COUNT(*) AS EXPR$4, SUM(d) AS EXPR$5, MAX(d) FILTER $f5 AS EXPR$6, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[b, a]])
+ +- Calc(select=[b, a, d, IS TRUE(>(b, 1000)) AS $f5, e, c, rowtime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
+
+== Optimized Execution Plan ==
+Calc(select=[a, b, window_start, window_end, EXPR$4, EXPR$5, EXPR$6, wAvg, uv])
++- WindowAggregate(groupBy=[b, a], window=[SESSION(time_col=[rowtime], gap=[5 min], partition keys=[b, a])], select=[b, a, COUNT(*) AS EXPR$4, SUM(d) AS EXPR$5, MAX(d) FILTER $f5 AS EXPR$6, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[b, a]])
+ +- Calc(select=[b, a, d, (b > 1000) IS TRUE AS $f5, e, c, rowtime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[(rowtime - 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testSessionWindowWithTwoPartitionKeys[aggPhaseEnforcer=TWO_PHASE]">
+ <Resource name="explain">
+ <![CDATA[== Abstract Syntax Tree ==
+LogicalProject(a=[$1], b=[$0], window_start=[$2], window_end=[$3], EXPR$4=[$4], EXPR$5=[$5], EXPR$6=[$6], wAvg=[$7], uv=[$8])
++- LogicalAggregate(group=[{0, 1, 2, 3}], EXPR$4=[COUNT()], EXPR$5=[SUM($4)], EXPR$6=[MAX($4) FILTER $5], wAvg=[weightedAvg($0, $6)], uv=[COUNT(DISTINCT $7)])
+ +- LogicalProject(b=[$1], a=[$0], window_start=[$7], window_end=[$8], d=[$3], $f5=[IS TRUE(>($1, 1000))], e=[$4], c=[$2])
+ +- LogicalTableFunctionScan(invocation=[SESSION(PARTITION BY($1, $0), DESCRIPTOR($5), 300000:INTERVAL MINUTE)], rowType=[RecordType(INTEGER a, BIGINT b, VARCHAR(2147483647) c, DECIMAL(10, 3) d, BIGINT e, TIMESTAMP(3) *ROWTIME* rowtime, TIMESTAMP_LTZ(3) *PROCTIME* proctime, TIMESTAMP(3) window_start, TIMESTAMP(3) window_end, TIMESTAMP(3) *ROWTIME* window_time)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[$6])
+ +- LogicalWatermarkAssigner(rowtime=[rowtime], watermark=[-($5, 1000:INTERVAL SECOND)])
+ +- LogicalProject(a=[$0], b=[$1], c=[$2], d=[$3], e=[$4], rowtime=[$5], proctime=[PROCTIME()])
+ +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]])
+
+== Optimized Physical Plan ==
+Calc(select=[a, b, window_start, window_end, EXPR$4, EXPR$5, EXPR$6, wAvg, uv])
++- WindowAggregate(groupBy=[b, a], window=[SESSION(time_col=[rowtime], gap=[5 min], partition keys=[b, a])], select=[b, a, COUNT(*) AS EXPR$4, SUM(d) AS EXPR$5, MAX(d) FILTER $f5 AS EXPR$6, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[b, a]])
+ +- Calc(select=[b, a, d, IS TRUE(>(b, 1000)) AS $f5, e, c, rowtime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[-(rowtime, 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
+
+== Optimized Execution Plan ==
+Calc(select=[a, b, window_start, window_end, EXPR$4, EXPR$5, EXPR$6, wAvg, uv])
++- WindowAggregate(groupBy=[b, a], window=[SESSION(time_col=[rowtime], gap=[5 min], partition keys=[b, a])], select=[b, a, COUNT(*) AS EXPR$4, SUM(d) AS EXPR$5, MAX(d) FILTER $f5 AS EXPR$6, weightedAvg(b, e) AS wAvg, COUNT(DISTINCT c) AS uv, start('w$) AS window_start, end('w$) AS window_end])
+ +- Exchange(distribution=[hash[b, a]])
+ +- Calc(select=[b, a, d, (b > 1000) IS TRUE AS $f5, e, c, rowtime])
+ +- WatermarkAssigner(rowtime=[rowtime], watermark=[(rowtime - 1000:INTERVAL SECOND)])
+ +- TableSourceScan(table=[[default_catalog, default_database, MyTable]], fields=[a, b, c, d, e, rowtime])
]]>
</Resource>
</TestCase>
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.scala
index 85825edec5f..a7b44993a32 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/plan/stream/sql/agg/WindowAggregateTest.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.planner.plan.stream.sql.agg
import org.apache.flink.configuration.Configuration
-import org.apache.flink.table.api.{ExplainDetail, TableException}
+import org.apache.flink.table.api.{ExplainDetail, TableException, ValidationException}
import org.apache.flink.table.api.config.OptimizerConfigOptions
import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.{WeightedAvg, WeightedAvgWithMerge}
import org.apache.flink.table.planner.plan.utils.WindowEmitStrategy.{TABLE_EXEC_EMIT_EARLY_FIRE_DELAY, TABLE_EXEC_EMIT_EARLY_FIRE_ENABLED, TABLE_EXEC_EMIT_LATE_FIRE_DELAY, TABLE_EXEC_EMIT_LATE_FIRE_ENABLED}
@@ -1433,6 +1433,230 @@ class WindowAggregateTest(aggPhaseEnforcer: AggregatePhaseStrategy) extends Tabl
|GROUP BY a, ws, we
""".stripMargin)
}
+
+ @TestTemplate
+ def testSession_OnRowtime(): Unit = {
+ // Session window does not support two-phase optimization
+ val sql =
+ """
+ |SELECT
+ | a,
+ | window_start,
+ | window_end,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | weightedAvg(b, e) AS wAvg,
+ | count(distinct c) AS uv
+ |FROM TABLE(
+ | SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ |GROUP BY a, window_start, window_end
+ """.stripMargin
+ util.verifyRelPlan(sql)
+ }
+
+ @TestTemplate
+ def testSession_OnProctime(): Unit = {
+ assumeThat(isTwoPhase).isTrue
+ val sql =
+ """
+ |SELECT
+ | a,
+ | window_start,
+ | window_end,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | weightedAvg(b, e) AS wAvg,
+ | count(distinct c) AS uv
+ |FROM TABLE(
+ | SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(proctime), INTERVAL '5' MINUTE))
+ |GROUP BY a, window_start, window_end
+ """.stripMargin
+ util.verifyRelPlan(sql)
+ }
+
+ @TestTemplate
+ def testSession_DistinctSplitEnabled(): Unit = {
+ // Session window does not support split-distinct optimization
+ util.tableEnv.getConfig.getConfiguration
+ .setBoolean(OptimizerConfigOptions.TABLE_OPTIMIZER_DISTINCT_AGG_SPLIT_ENABLED, true)
+ val sql =
+ """
+ |SELECT
+ | a,
+ | window_start,
+ | window_end,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | count(distinct c) AS uv
+ |FROM TABLE(
+ | SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(proctime), INTERVAL '5' MINUTE))
+ |GROUP BY a, window_start, window_end
+ """.stripMargin
+ util.verifyRelPlan(sql)
+ }
+
+ @TestTemplate
+ def testSessionWindowWithTwoPartitionKeys(): Unit = {
+ val sql =
+ """
+ |SELECT
+ | a,
+ | b,
+ | window_start,
+ | window_end,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | weightedAvg(b, e) AS wAvg,
+ | count(distinct c) AS uv
+ |FROM TABLE(
+ | SESSION(TABLE MyTable PARTITION BY (b, a), DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ |GROUP BY b, a, window_start, window_end
+ """.stripMargin
+
+ util.verifyExplain(sql)
+ }
+
+ @TestTemplate
+ def testGroupKeyMoreThanPartitionKeyInSessionWindow(): Unit = {
+ // the aggregate will not be converted to window aggregate
+ // TODO Support session window table function in ExecWindowTableFunction. See
+ // more at FLINK-34100
+ val sql =
+ """
+ |SELECT
+ | a,
+ | window_start,
+ | window_end,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | weightedAvg(b, e) AS wAvg,
+ | count(distinct c) AS uv
+ |FROM TABLE(
+ | SESSION(TABLE MyTable, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ |GROUP BY a, window_start, window_end
+ """.stripMargin
+
+ assertThatThrownBy(() => util.verifyExplain(sql))
+ .hasMessageContaining("Session Window TableFunction is not supported yet.")
+ .isInstanceOf[TableException]
+ }
+
+ @TestTemplate
+ def testGroupKeyLessThanPartitionKeyInSessionWindow(): Unit = {
+ // TODO Support session window table function in ExecWindowTableFunction. See
+ // more at FLINK-34100
+ val sql = {
+ """
+ |SELECT
+ | window_start,
+ | window_end,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | weightedAvg(b, e) AS wAvg,
+ | count(distinct c) AS uv
+ |FROM TABLE(
+ | SESSION(TABLE MyTable PARTITION BY (b, a), DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ |GROUP BY b, window_start, window_end
+ """.stripMargin
+ }
+
+ assertThatThrownBy(() => util.verifyExplain(sql))
+ .hasMessageContaining("Session Window TableFunction is not supported yet.")
+ .isInstanceOf[TableException]
+ }
+
+ @TestTemplate
+ def testDeprecatedSyntaxAboutPartitionKeyInSessionWindow(): Unit = {
+ // Session window does not support two-phase optimization
+ val sql =
+ """
+ |SELECT
+ | a,
+ | window_start,
+ | window_end,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | weightedAvg(b, e) AS wAvg,
+ | count(distinct c) AS uv
+ |FROM TABLE(
+ | SESSION(TABLE MyTable, DESCRIPTOR(proctime), DESCRIPTOR(a), INTERVAL '5' MINUTE))
+ |GROUP BY a, window_start, window_end
+ """.stripMargin
+
+ assertThatThrownBy(() => util.verifyExplain(sql))
+ .hasMessageContaining(
+ "Invalid number of arguments to function 'SESSION'. Was expecting 3 arguments")
+ .isInstanceOf[ValidationException]
+ }
+
+ @TestTemplate
+ def testGroupKeysIndicesChangesInSessionWindow(): Unit = {
+ val sql =
+ """
+ |SELECT
+ | window_start,
+ | window_end,
+ | count(*),
+ | a
+ |FROM TABLE(
+ | SESSION(TABLE MyTable partition by a, DESCRIPTOR(proctime), INTERVAL '10' MINUTE))
+ |GROUP BY window_start, window_end, a
+ """.stripMargin
+
+ util.verifyExplain(sql)
+ }
+
+ @TestTemplate
+ def testSessionWindowTVFWithPartitionKeyWhenCantMerge(): Unit = {
+ val sql =
+ """
+ |SELECT
+ | window_start,
+ | window_end,
+ | a,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | weightedAvg(b, e) AS wAvg,
+ | count(distinct c) AS uv
+ |FROM (
+ | SELECT window_start, rowtime, d, proctime, e, b, c, window_end, window_time, a
+ | FROM TABLE(SESSION(TABLE MyTable PARTITION BY a, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ | WHERE window_start >= TIMESTAMP '2021-01-01 10:10:00.000'
+ |)
+ |GROUP BY a, window_start, window_end
+ """.stripMargin
+ util.verifyRelPlan(sql)
+ }
+
+ @TestTemplate
+ def testSessionWindowTVFWithoutPartitionKeyWhenCantMerge(): Unit = {
+ val sql =
+ """
+ |SELECT
+ | window_start,
+ | window_end,
+ | count(*),
+ | sum(d),
+ | max(d) filter (where b > 1000),
+ | weightedAvg(b, e) AS wAvg,
+ | count(distinct c) AS uv
+ |FROM (
+ | SELECT window_start, rowtime, d, proctime, e, b, c, window_end, window_time, a
+ | FROM TABLE(SESSION(TABLE MyTable, DESCRIPTOR(rowtime), INTERVAL '5' MINUTE))
+ | WHERE window_start >= TIMESTAMP '2021-01-01 10:10:00.000'
+ |)
+ |GROUP BY window_start, window_end
+ """.stripMargin
+ util.verifyRelPlan(sql)
+ }
}
object WindowAggregateTest {
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowAggregateITCase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowAggregateITCase.scala
index 6d64a825104..f9c30366dea 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowAggregateITCase.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/runtime/stream/sql/WindowAggregateITCase.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.planner.runtime.stream.sql
import org.apache.flink.api.common.restartstrategy.RestartStrategies
import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.CheckpointingMode
+import org.apache.flink.table.api._
import org.apache.flink.table.api.bridge.scala._
import org.apache.flink.table.api.config.OptimizerConfigOptions
import org.apache.flink.table.planner.factories.TestValuesTableFactory
@@ -27,6 +28,7 @@ import org.apache.flink.table.planner.factories.TestValuesTableFactory.changelog
import org.apache.flink.table.planner.plan.utils.JavaUserDefinedAggFunctions.ConcatDistinctAggFunction
import org.apache.flink.table.planner.runtime.utils._
import org.apache.flink.table.planner.runtime.utils.StreamingWithStateTestBase.{HEAP_BACKEND, ROCKSDB_BACKEND, StateBackendMode}
+import org.apache.flink.table.planner.runtime.utils.TimeTestUtil.TimestampAndWatermarkWithOffset
import org.apache.flink.table.planner.utils.AggregatePhaseStrategy
import org.apache.flink.table.planner.utils.AggregatePhaseStrategy._
import org.apache.flink.testutils.junit.extensions.parameterized.{ParameterizedTestExtension, Parameters}
@@ -1180,6 +1182,112 @@ class WindowAggregateITCase(
assertThat(sink.getAppendResults.sorted.mkString("\n"))
.isEqualTo(expected.sorted.mkString("\n"))
}
+
+ @TestTemplate
+ def testEventTimeSessionWindow(): Unit = {
+ val sql =
+ """
+ |SELECT
+ | `name`,
+ | window_start,
+ | window_end,
+ | COUNT(*),
+ | SUM(`bigdec`),
+ | MAX(`double`),
+ | MIN(`float`),
+ | COUNT(DISTINCT `string`),
+ | concat_distinct_agg(`string`)
+ |FROM TABLE(
+ | SESSION(TABLE T1 PARTITION BY `name`, DESCRIPTOR(rowtime), INTERVAL '5' SECOND))
+ |GROUP BY `name`, window_start, window_end
+ """.stripMargin
+
+ val sink = new TestingAppendSink
+ tEnv.sqlQuery(sql).toDataStream.addSink(sink)
+ env.execute()
+
+ val expected = Seq(
+ "a,2020-10-10T00:00:01,2020-10-10T00:00:13,6,19.98,5.0,1.0,3,Hi|Comment#1|Comment#2",
+ "b,2020-10-10T00:00:06,2020-10-10T00:00:12,2,6.66,6.0,3.0,2,Hello|Hi",
+ "b,2020-10-10T00:00:16,2020-10-10T00:00:21,1,4.44,4.0,4.0,1,Hi",
+ "b,2020-10-10T00:00:34,2020-10-10T00:00:39,1,3.33,3.0,3.0,1,Comment#3",
+ "null,2020-10-10T00:00:32,2020-10-10T00:00:37,1,7.77,7.0,7.0,0,null"
+ )
+ assertThat(sink.getAppendResults.sorted.mkString("\n"))
+ .isEqualTo(expected.sorted.mkString("\n"))
+ }
+
+ @TestTemplate
+ def testEventTimeSessionWindowWithCDCSource(): Unit = {
+ val sql =
+ """
+ |SELECT
+ | `name`,
+ | window_start,
+ | window_end,
+ | COUNT(*),
+ | SUM(`bigdec`),
+ | MAX(`double`),
+ | MIN(`float`),
+ | COUNT(DISTINCT `string`)
+ |FROM TABLE(
+ | SESSION(TABLE T1_CDC PARTITION BY `name`, DESCRIPTOR(rowtime), INTERVAL '5' SECOND))
+ |GROUP BY `name`, window_start, window_end
+ """.stripMargin
+
+ val sink = new TestingAppendSink
+ tEnv.sqlQuery(sql).toDataStream.addSink(sink)
+ env.execute()
+
+ val expected = Seq(
+ "a,2020-10-10T00:00:01,2020-10-10T00:00:13,5,38.87,22.0,2.0,4",
+ "b,2020-10-10T00:00:06,2020-10-10T00:00:12,2,6.66,6.0,3.0,2",
+ "b,2020-10-10T00:00:16,2020-10-10T00:00:21,1,4.44,4.0,4.0,1"
+ )
+ assertThat(sink.getAppendResults.sorted.mkString("\n"))
+ .isEqualTo(expected.sorted.mkString("\n"))
+ }
+
+ @TestTemplate
+ def testDistinctAggWithMergeOnEventTimeSessionWindow(): Unit = {
+ // create a watermark with 10ms offset to delay the window emission by 10ms to verify merge
+ val sessionWindowTestData = List(
+ (1L, 2, "Hello"), // (1, Hello) - window
+ (2L, 2, "Hello"), // (1, Hello) - window, deduped
+ (8L, 2, "Hello"), // (2, Hello) - window, deduped during merge
+ (10L, 3, "Hello"), // (2, Hello) - window, forwarded during merge
+ (9L, 9, "Hello World"), // (1, Hello World) - window
+ (4L, 1, "Hello"), // (1, Hello) - window, triggering merge
+ (16L, 16, "Hello")
+ ) // (3, Hello) - window (not merged)
+
+ val stream = failingDataSource(sessionWindowTestData)
+ .assignTimestampsAndWatermarks(new TimestampAndWatermarkWithOffset[(Long, Int, String)](10L))
+ val table = stream.toTable(tEnv, 'a, 'b, 'c, 'rowtime.rowtime)
+ tEnv.registerTable("MyTable", table)
+
+ val sqlQuery =
+ """
+ |SELECT c,
+ | COUNT(DISTINCT b),
+ | window_end
+ |FROM TABLE(
+ | SESSION(TABLE MyTable PARTITION BY c, DESCRIPTOR(rowtime), INTERVAL '0.005' SECOND))
+ |GROUP BY c, window_start, window_end
+ """.stripMargin
+ val sink = new TestingAppendSink
+ tEnv.sqlQuery(sqlQuery).toAppendStream[Row].addSink(sink)
+ env.execute()
+
+ val expected = Seq(
+ "Hello World,1,1970-01-01T00:00:00.014", // window starts at [9L] till {14L}
+ "Hello,1,1970-01-01T00:00:00.021", // window starts at [16L] till {21L}, not merged
+ "Hello,3,1970-01-01T00:00:00.015" // window starts at [1L,2L],
+ // merged with [8L,10L], by [4L], till {15L}
+ )
+ assertThat(sink.getAppendResults.sorted.mkString("\n"))
+ .isEqualTo(expected.sorted.mkString("\n"))
+ }
}
object WindowAggregateITCase {