You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ro...@apache.org on 2024/01/25 09:33:45 UTC
(flink) 03/03: [FLINK-34048][table] introduce unslicing window agg for session window tvf
This is an automated email from the ASF dual-hosted git repository.
ron pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit de7322112cbf594ae4b74163b5e58892e6cdf4ca
Author: xuyang <xy...@163.com>
AuthorDate: Fri Jan 19 16:37:06 2024 +0800
[FLINK-34048][table] introduce unslicing window agg for session window tvf
This closes #24150
---
.../stream/StreamExecGlobalWindowAggregate.java | 14 +-
.../stream/StreamExecLocalWindowAggregate.java | 10 +-
.../exec/stream/StreamExecWindowAggregate.java | 88 +--
.../exec/stream/StreamExecWindowAggregateBase.java | 36 +-
.../codegen/agg/AggsHandlerCodeGenerator.scala | 20 +-
...rBuilder.java => WindowAggOperatorBuilder.java} | 106 +++-
.../AbstractSliceWindowAggProcessor.java | 2 +-
.../processors/SliceSharedWindowAggProcessor.java | 2 +-
.../SliceUnsharedWindowAggProcessor.java | 2 +-
.../processors/UnsliceWindowAggProcessor.java | 391 +++++++++++++
.../RowTimeWindowDeduplicateProcessor.java | 4 +-
.../window/processors/WindowRankProcessor.java | 4 +-
.../window/tvf/common/WindowOperatorBase.java | 4 +-
.../window/tvf/common/WindowProcessor.java | 6 +-
.../window/SlicingWindowAggOperatorTest.java | 351 +++---------
.../window/UnslicingWindowAggOperatorTest.java | 628 +++++++++++++++++++++
.../window/WindowAggOperatorTestBase.java | 278 +++++++++
17 files changed, 1549 insertions(+), 397 deletions(-)
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecGlobalWindowAggregate.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecGlobalWindowAggregate.java
index a2138db1efd..303444ae3d7 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecGlobalWindowAggregate.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecGlobalWindowAggregate.java
@@ -45,7 +45,8 @@ import org.apache.flink.table.runtime.generated.GeneratedNamespaceAggsHandleFunc
import org.apache.flink.table.runtime.groupwindow.NamedWindowProperty;
import org.apache.flink.table.runtime.groupwindow.WindowProperty;
import org.apache.flink.table.runtime.keyselector.RowDataKeySelector;
-import org.apache.flink.table.runtime.operators.aggregate.window.SlicingWindowAggOperatorBuilder;
+import org.apache.flink.table.runtime.operators.aggregate.window.WindowAggOperatorBuilder;
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigner;
import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
import org.apache.flink.table.runtime.typeutils.PagedTypeSerializer;
@@ -70,6 +71,7 @@ import java.util.List;
import java.util.Optional;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
/** Stream {@link ExecNode} for window table-valued based global aggregate. */
@ExecNodeMetadata(
@@ -169,7 +171,11 @@ public class StreamExecGlobalWindowAggregate extends StreamExecWindowAggregateBa
TimeWindowUtil.getShiftTimeZone(
windowing.getTimeAttributeType(),
TableConfigUtils.getLocalTimeZone(config));
- final SliceAssigner sliceAssigner = createSliceAssigner(windowing, shiftTimeZone);
+ final WindowAssigner windowAssigner = createWindowAssigner(windowing, shiftTimeZone);
+ checkState(
+ windowAssigner instanceof SliceAssigner,
+ "Two-stage optimization on unslicing window like SESSION are not supported yet");
+ final SliceAssigner sliceAssigner = (SliceAssigner) windowAssigner;
final AggregateInfoList localAggInfoList =
AggregateUtil.deriveStreamWindowAggregateInfoList(
@@ -241,7 +247,7 @@ public class StreamExecGlobalWindowAggregate extends StreamExecWindowAggregateBa
final LogicalType[] accTypes = convertToLogicalTypes(globalAggInfoList.getAccTypes());
final OneInputStreamOperator<RowData, RowData> windowOperator =
- SlicingWindowAggOperatorBuilder.builder()
+ WindowAggOperatorBuilder.builder()
.inputSerializer(new RowDataSerializer(inputRowType))
.shiftTimeZone(shiftTimeZone)
.keySerializer(
@@ -303,6 +309,8 @@ public class StreamExecGlobalWindowAggregate extends StreamExecWindowAggregateBa
aggInfoList,
JavaScalaConversionUtil.toScala(windowProperties),
sliceAssigner,
+ // we use window end timestamp to indicate a slicing window, see SliceAssigner
+ Long.class,
shifTimeZone);
}
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLocalWindowAggregate.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLocalWindowAggregate.java
index 049a48ce85b..93a512d8a1f 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLocalWindowAggregate.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecLocalWindowAggregate.java
@@ -46,6 +46,7 @@ import org.apache.flink.table.runtime.operators.aggregate.window.LocalSlicingWin
import org.apache.flink.table.runtime.operators.aggregate.window.buffers.RecordsWindowBuffer;
import org.apache.flink.table.runtime.operators.aggregate.window.buffers.WindowBuffer;
import org.apache.flink.table.runtime.operators.aggregate.window.combines.LocalAggCombiner;
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigner;
import org.apache.flink.table.runtime.typeutils.AbstractRowDataSerializer;
import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
@@ -70,6 +71,7 @@ import java.util.List;
import java.util.Optional;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
/** Stream {@link ExecNode} for window table-valued based local aggregate. */
@ExecNodeMetadata(
@@ -155,7 +157,11 @@ public class StreamExecLocalWindowAggregate extends StreamExecWindowAggregateBas
TimeWindowUtil.getShiftTimeZone(
windowing.getTimeAttributeType(),
TableConfigUtils.getLocalTimeZone(config));
- final SliceAssigner sliceAssigner = createSliceAssigner(windowing, shiftTimeZone);
+ final WindowAssigner windowAssigner = createWindowAssigner(windowing, shiftTimeZone);
+ checkState(
+ windowAssigner instanceof SliceAssigner,
+ "Two-stage optimization on unslicing window like SESSION are not supported yet");
+ final SliceAssigner sliceAssigner = (SliceAssigner) windowAssigner;
final AggregateInfoList aggInfoList =
AggregateUtil.deriveStreamWindowAggregateInfoList(
@@ -230,6 +236,8 @@ public class StreamExecLocalWindowAggregate extends StreamExecWindowAggregateBas
aggInfoList,
JavaScalaConversionUtil.toScala(Collections.emptyList()),
sliceAssigner,
+ // we use window end timestamp to indicate a slicing window, see SliceAssigner
+ Long.class,
shiftTimeZone);
}
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregate.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregate.java
index fafc27b20da..dcf8223c410 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregate.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregate.java
@@ -25,14 +25,9 @@ import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.SimpleOperatorFactory;
import org.apache.flink.streaming.api.transformations.OneInputTransformation;
import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.expressions.FieldReferenceExpression;
import org.apache.flink.table.planner.codegen.CodeGeneratorContext;
import org.apache.flink.table.planner.codegen.agg.AggsHandlerCodeGenerator;
import org.apache.flink.table.planner.delegation.PlannerBase;
-import org.apache.flink.table.planner.plan.logical.LogicalWindow;
-import org.apache.flink.table.planner.plan.logical.SessionGroupWindow;
-import org.apache.flink.table.planner.plan.logical.SessionWindowSpec;
-import org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy;
import org.apache.flink.table.planner.plan.logical.WindowingStrategy;
import org.apache.flink.table.planner.plan.nodes.exec.ExecEdge;
import org.apache.flink.table.planner.plan.nodes.exec.ExecNode;
@@ -49,10 +44,10 @@ import org.apache.flink.table.planner.utils.TableConfigUtils;
import org.apache.flink.table.runtime.generated.GeneratedNamespaceAggsHandleFunction;
import org.apache.flink.table.runtime.groupwindow.NamedWindowProperty;
import org.apache.flink.table.runtime.groupwindow.WindowProperty;
-import org.apache.flink.table.runtime.groupwindow.WindowReference;
import org.apache.flink.table.runtime.keyselector.RowDataKeySelector;
-import org.apache.flink.table.runtime.operators.aggregate.window.SlicingWindowAggOperatorBuilder;
-import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigner;
+import org.apache.flink.table.runtime.operators.aggregate.window.WindowAggOperatorBuilder;
+import org.apache.flink.table.runtime.operators.window.TimeWindow;
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceSharedAssigner;
import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
import org.apache.flink.table.runtime.typeutils.PagedTypeSerializer;
@@ -60,7 +55,6 @@ import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
import org.apache.flink.table.runtime.util.TimeWindowUtil;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.util.Preconditions;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
@@ -76,8 +70,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Optional;
-import static org.apache.flink.table.expressions.ApiExpressionUtils.intervalOfMillis;
-import static org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter.fromLogicalTypeToDataType;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
@@ -169,12 +161,6 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
@Override
protected Transformation<RowData> translateToPlanInternal(
PlannerBase planner, ExecNodeConfig config) {
- // TODO Currently, the operator of WindowAggregate does not support Session Window, and it
- // needs to fall back to the legacy GroupWindowAggregate. See more at FLINK-34048.
- if (windowing.getWindow() instanceof SessionWindowSpec) {
- return fallbackToLegacyGroupWindowAggregate(planner, config);
- }
-
final ExecEdge inputEdge = getInputEdges().get(0);
final Transformation<RowData> inputTransform =
(Transformation<RowData>) inputEdge.translateToPlan(planner);
@@ -184,7 +170,7 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
TimeWindowUtil.getShiftTimeZone(
windowing.getTimeAttributeType(),
TableConfigUtils.getLocalTimeZone(config));
- final SliceAssigner sliceAssigner = createSliceAssigner(windowing, shiftTimeZone);
+ final WindowAssigner windowAssigner = createWindowAssigner(windowing, shiftTimeZone);
final AggregateInfoList aggInfoList =
AggregateUtil.deriveStreamWindowAggregateInfoList(
@@ -195,9 +181,9 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
windowing.getWindow(),
true); // isStateBackendDataViews
- final GeneratedNamespaceAggsHandleFunction<Long> generatedAggsHandler =
+ final GeneratedNamespaceAggsHandleFunction<?> generatedAggsHandler =
createAggsHandler(
- sliceAssigner,
+ windowAssigner,
aggInfoList,
config,
planner.getFlinkContext().getClassLoader(),
@@ -213,13 +199,13 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
final LogicalType[] accTypes = convertToLogicalTypes(aggInfoList.getAccTypes());
final OneInputStreamOperator<RowData, RowData> windowOperator =
- SlicingWindowAggOperatorBuilder.builder()
+ WindowAggOperatorBuilder.builder()
.inputSerializer(new RowDataSerializer(inputRowType))
.shiftTimeZone(shiftTimeZone)
.keySerializer(
(PagedTypeSerializer<RowData>)
selector.getProducedType().toSerializer())
- .assigner(sliceAssigner)
+ .assigner(windowAssigner)
.countStarIndex(aggInfoList.getIndexOfCountStar())
.aggregate(generatedAggsHandler, new RowDataSerializer(accTypes))
.build();
@@ -240,8 +226,8 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
return transform;
}
- private GeneratedNamespaceAggsHandleFunction<Long> createAggsHandler(
- SliceAssigner sliceAssigner,
+ private GeneratedNamespaceAggsHandleFunction<?> createAggsHandler(
+ WindowAssigner windowAssigner,
AggregateInfoList aggInfoList,
ExecNodeConfig config,
ClassLoader classLoader,
@@ -256,7 +242,8 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
false) // copyInputField
.needAccumulate();
- if (sliceAssigner instanceof SliceSharedAssigner) {
+ if (windowAssigner instanceof SliceSharedAssigner
+ || !isAlignedWindow(windowing.getWindow())) {
generator.needMerge(0, false, null);
}
@@ -270,54 +257,17 @@ public class StreamExecWindowAggregate extends StreamExecWindowAggregateBase {
.map(NamedWindowProperty::getProperty)
.toArray(WindowProperty[]::new));
+ // We use window end timestamp to indicate a slicing window, see SliceAssigner. And
+ // use window start and window end to indicate a unslicing window, see UnsliceAssigner
+ final Class<?> windowClass =
+ isAlignedWindow(windowing.getWindow()) ? Long.class : TimeWindow.class;
+
return generator.generateNamespaceAggsHandler(
"WindowAggsHandler",
aggInfoList,
JavaScalaConversionUtil.toScala(windowProperties),
- sliceAssigner,
+ windowAssigner,
+ windowClass,
shiftTimeZone);
}
-
- private Transformation<RowData> fallbackToLegacyGroupWindowAggregate(
- PlannerBase planner, ExecNodeConfig config) {
- Preconditions.checkState(windowing.getWindow() instanceof SessionWindowSpec);
-
- if (windowing instanceof TimeAttributeWindowingStrategy) {
- LogicalType timeAttributeType = windowing.getTimeAttributeType();
- LogicalWindow logicalWindow =
- new SessionGroupWindow(
- new WindowReference("w$", timeAttributeType),
- new FieldReferenceExpression(
- // mock an empty time field name here
- "",
- fromLogicalTypeToDataType(timeAttributeType),
- 0,
- ((TimeAttributeWindowingStrategy) windowing)
- .getTimeAttributeIndex()),
- intervalOfMillis(
- ((SessionWindowSpec) windowing.getWindow())
- .getGap()
- .toMillis()));
-
- StreamExecGroupWindowAggregate groupWindowAggregate =
- new StreamExecGroupWindowAggregate(
- planner.getTableConfig(),
- grouping,
- aggCalls,
- logicalWindow,
- namedWindowProperties,
- needRetraction,
- InputProperty.DEFAULT,
- (RowType) getOutputType(),
- getDescription());
-
- groupWindowAggregate.setInputEdges(getInputEdges());
- return groupWindowAggregate.translateToPlanInternal(planner, config);
- }
-
- throw new UnsupportedOperationException(
- String.format(
- "Unsupported windowing strategy: %s for session window.",
- windowing.getClass()));
- }
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregateBase.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregateBase.java
index e34f91edf48..c3b515dc743 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregateBase.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecWindowAggregateBase.java
@@ -22,6 +22,7 @@ import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.table.api.TableException;
import org.apache.flink.table.planner.plan.logical.CumulativeWindowSpec;
import org.apache.flink.table.planner.plan.logical.HoppingWindowSpec;
+import org.apache.flink.table.planner.plan.logical.SessionWindowSpec;
import org.apache.flink.table.planner.plan.logical.SliceAttachedWindowingStrategy;
import org.apache.flink.table.planner.plan.logical.TimeAttributeWindowingStrategy;
import org.apache.flink.table.planner.plan.logical.TumblingWindowSpec;
@@ -31,8 +32,12 @@ import org.apache.flink.table.planner.plan.logical.WindowingStrategy;
import org.apache.flink.table.planner.plan.nodes.exec.ExecNode;
import org.apache.flink.table.planner.plan.nodes.exec.ExecNodeContext;
import org.apache.flink.table.planner.plan.nodes.exec.InputProperty;
+import org.apache.flink.table.runtime.operators.window.TimeWindow;
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigners;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceAssigner;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceAssigners;
import org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.logical.LogicalType;
@@ -66,10 +71,14 @@ public abstract class StreamExecWindowAggregateBase extends StreamExecAggregateB
// Utilities
// ------------------------------------------------------------------------------------------
- protected SliceAssigner createSliceAssigner(
+ protected WindowAssigner createWindowAssigner(
WindowingStrategy windowingStrategy, ZoneId shiftTimeZone) {
WindowSpec windowSpec = windowingStrategy.getWindow();
if (windowingStrategy instanceof WindowAttachedWindowingStrategy) {
+ checkArgument(
+ isAlignedWindow(windowSpec),
+ "UnsliceAssigner with WindowAttachedWindowingStrategy is not supported yet.");
+
int windowEndIndex =
((WindowAttachedWindowingStrategy) windowingStrategy).getWindowEnd();
// we don't need time attribute to assign windows, use a magic value in this case
@@ -78,6 +87,10 @@ public abstract class StreamExecWindowAggregateBase extends StreamExecAggregateB
return SliceAssigners.windowed(windowEndIndex, innerAssigner);
} else if (windowingStrategy instanceof SliceAttachedWindowingStrategy) {
+ checkArgument(
+ isAlignedWindow(windowSpec),
+ "UnsliceAssigner with SliceAttachedWindowingStrategy is not supported yet.");
+
int sliceEndIndex = ((SliceAttachedWindowingStrategy) windowingStrategy).getSliceEnd();
// we don't need time attribute to assign windows, use a magic value in this case
SliceAssigner innerAssigner =
@@ -93,14 +106,22 @@ public abstract class StreamExecWindowAggregateBase extends StreamExecAggregateB
} else {
timeAttributeIndex = -1;
}
- return createSliceAssigner(windowSpec, timeAttributeIndex, shiftTimeZone);
+ if (isAlignedWindow(windowSpec)) {
+ return createSliceAssigner(windowSpec, timeAttributeIndex, shiftTimeZone);
+ } else {
+ return createUnsliceAssigner(windowSpec, timeAttributeIndex, shiftTimeZone);
+ }
} else {
throw new UnsupportedOperationException(windowingStrategy + " is not supported yet.");
}
}
- protected SliceAssigner createSliceAssigner(
+ protected boolean isAlignedWindow(WindowSpec window) {
+ return window.isAlignedWindow();
+ }
+
+ private SliceAssigner createSliceAssigner(
WindowSpec windowSpec, int timeAttributeIndex, ZoneId shiftTimeZone) {
if (windowSpec instanceof TumblingWindowSpec) {
Duration size = ((TumblingWindowSpec) windowSpec).getSize();
@@ -150,6 +171,15 @@ public abstract class StreamExecWindowAggregateBase extends StreamExecAggregateB
}
}
+ private UnsliceAssigner<TimeWindow> createUnsliceAssigner(
+ WindowSpec windowSpec, int timeAttributeIndex, ZoneId shiftTimeZone) {
+ if (windowSpec instanceof SessionWindowSpec) {
+ Duration gap = ((SessionWindowSpec) windowSpec).getGap();
+ return UnsliceAssigners.session(timeAttributeIndex, shiftTimeZone, gap);
+ }
+ throw new UnsupportedOperationException(windowSpec + " is not supported yet.");
+ }
+
protected LogicalType[] convertToLogicalTypes(DataType[] dataTypes) {
return Arrays.stream(dataTypes)
.map(LogicalTypeDataTypeConverter::fromDataTypeToLogicalType)
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/AggsHandlerCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/AggsHandlerCodeGenerator.scala
index 2f67da5fbf3..c9c7ee5d500 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/AggsHandlerCodeGenerator.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/agg/AggsHandlerCodeGenerator.scala
@@ -35,7 +35,7 @@ import org.apache.flink.table.planner.utils.JavaScalaConversionUtil.toScala
import org.apache.flink.table.runtime.dataview._
import org.apache.flink.table.runtime.generated._
import org.apache.flink.table.runtime.groupwindow._
-import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigner
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowAssigner
import org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter.fromDataTypeToLogicalType
import org.apache.flink.table.types.DataType
import org.apache.flink.table.types.logical.{BooleanType, IntType, LogicalType, RowType}
@@ -71,7 +71,7 @@ class AggsHandlerCodeGenerator(
private var namespaceClassName: String = _
private var windowProperties: Seq[WindowProperty] = Seq()
private var hasNamespace: Boolean = false
- private var sliceAssignerTerm: String = _
+ private var windowAssignerTerm: String = _
private var shiftTimeZone: ZoneId = _
/** Aggregates informations */
@@ -626,16 +626,16 @@ class AggsHandlerCodeGenerator(
* Generate [[NamespaceAggsHandleFunction]] with the given function name and aggregate infos and
* window properties.
*/
- def generateNamespaceAggsHandler(
+ def generateNamespaceAggsHandler[N](
name: String,
aggInfoList: AggregateInfoList,
windowProperties: Seq[WindowProperty],
- sliceAssigner: SliceAssigner,
- shiftTimeZone: ZoneId): GeneratedNamespaceAggsHandleFunction[JLong] = {
- this.sliceAssignerTerm = newName(ctx, "sliceAssigner")
- ctx.addReusableObjectWithName(sliceAssigner, sliceAssignerTerm)
- // we use window end timestamp to indicate a window, see SliceAssigner
- generateNamespaceAggsHandler(name, aggInfoList, windowProperties, classOf[JLong], shiftTimeZone)
+ windowAssigner: WindowAssigner,
+ windowClass: Class[N],
+ shiftTimeZone: ZoneId): GeneratedNamespaceAggsHandleFunction[N] = {
+ this.windowAssignerTerm = newName(ctx, "windowAssigner")
+ ctx.addReusableObjectWithName(windowAssigner, windowAssignerTerm)
+ generateNamespaceAggsHandler(name, aggInfoList, windowProperties, windowClass, shiftTimeZone)
}
/**
@@ -1097,7 +1097,7 @@ class AggsHandlerCodeGenerator(
case w: WindowStart =>
// return a Timestamp(Internal is TimestampData)
GeneratedExpression(
- s"$TIMESTAMP_DATA.fromEpochMillis($sliceAssignerTerm.getWindowStart($NAMESPACE_TERM))",
+ s"$TIMESTAMP_DATA.fromEpochMillis($windowAssignerTerm.getWindowStart($NAMESPACE_TERM))",
"false",
"",
w.getResultType)
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/SlicingWindowAggOperatorBuilder.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/WindowAggOperatorBuilder.java
similarity index 58%
rename from flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/SlicingWindowAggOperatorBuilder.java
rename to flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/WindowAggOperatorBuilder.java
index f0e90e41bfb..7228a5f2fe1 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/SlicingWindowAggOperatorBuilder.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/WindowAggOperatorBuilder.java
@@ -26,13 +26,20 @@ import org.apache.flink.table.runtime.operators.aggregate.window.combines.AggCom
import org.apache.flink.table.runtime.operators.aggregate.window.combines.GlobalAggCombiner;
import org.apache.flink.table.runtime.operators.aggregate.window.processors.SliceSharedWindowAggProcessor;
import org.apache.flink.table.runtime.operators.aggregate.window.processors.SliceUnsharedWindowAggProcessor;
+import org.apache.flink.table.runtime.operators.aggregate.window.processors.UnsliceWindowAggProcessor;
+import org.apache.flink.table.runtime.operators.window.TimeWindow;
import org.apache.flink.table.runtime.operators.window.tvf.combines.RecordsCombiner;
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowAssigner;
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowOperatorBase;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigners.HoppingSliceAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceSharedAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceUnsharedAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SlicingWindowOperator;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SlicingWindowProcessor;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceAssigner;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceAssigners;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnslicingWindowOperator;
import org.apache.flink.table.runtime.typeutils.AbstractRowDataSerializer;
import org.apache.flink.table.runtime.typeutils.PagedTypeSerializer;
@@ -42,68 +49,78 @@ import java.util.function.Supplier;
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
- * The {@link SlicingWindowAggOperatorBuilder} is used to build a {@link SlicingWindowOperator} for
- * window aggregate.
+ * The {@link WindowAggOperatorBuilder} is used to build a {@link SlicingWindowOperator} with
+ * slicing window or a {@link UnslicingWindowOperator} for with unslicing window.
*
* <pre>
- * SlicingWindowAggOperatorBuilder.builder()
+ * WindowAggOperatorBuilder.builder()
* .inputType(inputType)
* .keyTypes(keyFieldTypes)
* .assigner(SliceAssigners.tumbling(rowtimeIndex, Duration.ofSeconds(5)))
* .aggregate(genAggsFunction), accTypes)
* .build();
* </pre>
+ *
+ * <p>or
+ *
+ * <pre>
+ * WindowAggOperatorBuilder.builder()
+ * .inputType(inputType)
+ * .keyTypes(keyFieldTypes)
+ * .assigner(UnsliceAssigners.session(rowtimeIndex, Duration.ofSeconds(5)))
+ * .aggregate(genAggsFunction), accTypes)
+ * .build();
+ * </pre>
*/
-public class SlicingWindowAggOperatorBuilder {
+public class WindowAggOperatorBuilder {
- public static SlicingWindowAggOperatorBuilder builder() {
- return new SlicingWindowAggOperatorBuilder();
+ public static WindowAggOperatorBuilder builder() {
+ return new WindowAggOperatorBuilder();
}
- private SliceAssigner assigner;
+ private WindowAssigner assigner;
private AbstractRowDataSerializer<RowData> inputSerializer;
private PagedTypeSerializer<RowData> keySerializer;
private AbstractRowDataSerializer<RowData> accSerializer;
- private GeneratedNamespaceAggsHandleFunction<Long> generatedAggregateFunction;
- private GeneratedNamespaceAggsHandleFunction<Long> localGeneratedAggregateFunction;
- private GeneratedNamespaceAggsHandleFunction<Long> globalGeneratedAggregateFunction;
+ private GeneratedNamespaceAggsHandleFunction<?> generatedAggregateFunction;
+ private GeneratedNamespaceAggsHandleFunction<?> localGeneratedAggregateFunction;
+ private GeneratedNamespaceAggsHandleFunction<?> globalGeneratedAggregateFunction;
private int indexOfCountStart = -1;
private ZoneId shiftTimeZone;
- public SlicingWindowAggOperatorBuilder inputSerializer(
+ public WindowAggOperatorBuilder inputSerializer(
AbstractRowDataSerializer<RowData> inputSerializer) {
this.inputSerializer = inputSerializer;
return this;
}
- public SlicingWindowAggOperatorBuilder shiftTimeZone(ZoneId shiftTimeZone) {
+ public WindowAggOperatorBuilder shiftTimeZone(ZoneId shiftTimeZone) {
this.shiftTimeZone = shiftTimeZone;
return this;
}
- public SlicingWindowAggOperatorBuilder keySerializer(
- PagedTypeSerializer<RowData> keySerializer) {
+ public WindowAggOperatorBuilder keySerializer(PagedTypeSerializer<RowData> keySerializer) {
this.keySerializer = keySerializer;
return this;
}
- public SlicingWindowAggOperatorBuilder assigner(SliceAssigner assigner) {
+ public WindowAggOperatorBuilder assigner(WindowAssigner assigner) {
this.assigner = assigner;
return this;
}
- public SlicingWindowAggOperatorBuilder aggregate(
- GeneratedNamespaceAggsHandleFunction<Long> generatedAggregateFunction,
+ public WindowAggOperatorBuilder aggregate(
+ GeneratedNamespaceAggsHandleFunction<?> generatedAggregateFunction,
AbstractRowDataSerializer<RowData> accSerializer) {
this.generatedAggregateFunction = generatedAggregateFunction;
this.accSerializer = accSerializer;
return this;
}
- public SlicingWindowAggOperatorBuilder globalAggregate(
- GeneratedNamespaceAggsHandleFunction<Long> localGeneratedAggregateFunction,
- GeneratedNamespaceAggsHandleFunction<Long> globalGeneratedAggregateFunction,
- GeneratedNamespaceAggsHandleFunction<Long> stateGeneratedAggregateFunction,
+ public WindowAggOperatorBuilder globalAggregate(
+ GeneratedNamespaceAggsHandleFunction<?> localGeneratedAggregateFunction,
+ GeneratedNamespaceAggsHandleFunction<?> globalGeneratedAggregateFunction,
+ GeneratedNamespaceAggsHandleFunction<?> stateGeneratedAggregateFunction,
AbstractRowDataSerializer<RowData> accSerializer) {
this.localGeneratedAggregateFunction = localGeneratedAggregateFunction;
this.globalGeneratedAggregateFunction = globalGeneratedAggregateFunction;
@@ -119,18 +136,28 @@ public class SlicingWindowAggOperatorBuilder {
*
* @see HoppingSliceAssigner#nextTriggerWindow(long, Supplier)
*/
- public SlicingWindowAggOperatorBuilder countStarIndex(int indexOfCountStart) {
+ public WindowAggOperatorBuilder countStarIndex(int indexOfCountStart) {
this.indexOfCountStart = indexOfCountStart;
return this;
}
- public SlicingWindowOperator<RowData, ?> build() {
+ public WindowOperatorBase<RowData, ?> build() {
checkNotNull(assigner);
checkNotNull(inputSerializer);
checkNotNull(keySerializer);
checkNotNull(accSerializer);
checkNotNull(generatedAggregateFunction);
+ if (assigner instanceof SliceAssigner) {
+ return buildSlicingWindowOperator();
+ } else {
+ return buildUnslicingWindowOperator();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected WindowOperatorBase<RowData, ?> buildSlicingWindowOperator() {
+
boolean isGlobalAgg =
localGeneratedAggregateFunction != null && globalGeneratedAggregateFunction != null;
@@ -138,9 +165,15 @@ public class SlicingWindowAggOperatorBuilder {
if (isGlobalAgg) {
combinerFactory =
new GlobalAggCombiner.Factory(
- localGeneratedAggregateFunction, globalGeneratedAggregateFunction);
+ (GeneratedNamespaceAggsHandleFunction<Long>)
+ localGeneratedAggregateFunction,
+ (GeneratedNamespaceAggsHandleFunction<Long>)
+ globalGeneratedAggregateFunction);
} else {
- combinerFactory = new AggCombiner.Factory(generatedAggregateFunction);
+ combinerFactory =
+ new AggCombiner.Factory(
+ (GeneratedNamespaceAggsHandleFunction<Long>)
+ generatedAggregateFunction);
}
final WindowBuffer.Factory bufferFactory =
new RecordsWindowBuffer.Factory(keySerializer, inputSerializer, combinerFactory);
@@ -149,7 +182,7 @@ public class SlicingWindowAggOperatorBuilder {
if (assigner instanceof SliceSharedAssigner) {
windowProcessor =
new SliceSharedWindowAggProcessor(
- generatedAggregateFunction,
+ (GeneratedNamespaceAggsHandleFunction<Long>) generatedAggregateFunction,
bufferFactory,
(SliceSharedAssigner) assigner,
accSerializer,
@@ -158,7 +191,7 @@ public class SlicingWindowAggOperatorBuilder {
} else if (assigner instanceof SliceUnsharedAssigner) {
windowProcessor =
new SliceUnsharedWindowAggProcessor(
- generatedAggregateFunction,
+ (GeneratedNamespaceAggsHandleFunction<Long>) generatedAggregateFunction,
bufferFactory,
(SliceUnsharedAssigner) assigner,
accSerializer,
@@ -170,4 +203,23 @@ public class SlicingWindowAggOperatorBuilder {
}
return new SlicingWindowOperator<>(windowProcessor);
}
+
+ @SuppressWarnings("unchecked")
+ private WindowOperatorBase<RowData, ?> buildUnslicingWindowOperator() {
+
+ if (assigner instanceof UnsliceAssigners.SessionUnsliceAssigner) {
+ final UnsliceWindowAggProcessor windowProcessor =
+ new UnsliceWindowAggProcessor(
+ (GeneratedNamespaceAggsHandleFunction<TimeWindow>)
+ generatedAggregateFunction,
+ (UnsliceAssigner<TimeWindow>) assigner,
+ accSerializer,
+ indexOfCountStart,
+ shiftTimeZone);
+ return new UnslicingWindowOperator<>(windowProcessor);
+ }
+
+ throw new UnsupportedOperationException(
+ "Unsupported unslice assigner: " + assigner.getClass().getCanonicalName());
+ }
}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/AbstractSliceWindowAggProcessor.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/AbstractSliceWindowAggProcessor.java
index 58423075bf9..19c727a5525 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/AbstractSliceWindowAggProcessor.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/AbstractSliceWindowAggProcessor.java
@@ -157,7 +157,7 @@ public abstract class AbstractSliceWindowAggProcessor extends AbstractWindowAggP
}
@Override
- public void clearWindow(Long windowEnd) throws Exception {
+ public void clearWindow(long timerTimestamp, Long windowEnd) throws Exception {
Iterable<Long> expires = sliceAssigner.expiredSlices(windowEnd);
for (Long slice : expires) {
windowState.clear(slice);
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceSharedWindowAggProcessor.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceSharedWindowAggProcessor.java
index 21aec23efe8..6dd076bf540 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceSharedWindowAggProcessor.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceSharedWindowAggProcessor.java
@@ -61,7 +61,7 @@ public final class SliceSharedWindowAggProcessor extends AbstractSliceWindowAggP
}
@Override
- public void fireWindow(Long windowEnd) throws Exception {
+ public void fireWindow(long timerTimestamp, Long windowEnd) throws Exception {
sliceSharedAssigner.mergeSlices(windowEnd, this);
// we have set accumulator in the merge() method
RowData aggResult = aggregator.getValue(windowEnd);
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceUnsharedWindowAggProcessor.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceUnsharedWindowAggProcessor.java
index 5476dfbffd9..20126e24e01 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceUnsharedWindowAggProcessor.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/SliceUnsharedWindowAggProcessor.java
@@ -50,7 +50,7 @@ public final class SliceUnsharedWindowAggProcessor extends AbstractSliceWindowAg
}
@Override
- public void fireWindow(Long windowEnd) throws Exception {
+ public void fireWindow(long timerTimestamp, Long windowEnd) throws Exception {
RowData acc = windowState.value(windowEnd);
if (acc == null) {
acc = aggregator.createAccumulators();
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/UnsliceWindowAggProcessor.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/UnsliceWindowAggProcessor.java
new file mode 100644
index 00000000000..33be17388db
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/aggregate/window/processors/UnsliceWindowAggProcessor.java
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.aggregate.window.processors;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.state.MergingState;
+import org.apache.flink.api.common.state.State;
+import org.apache.flink.api.common.state.StateDescriptor;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.state.VoidNamespace;
+import org.apache.flink.runtime.state.VoidNamespaceSerializer;
+import org.apache.flink.runtime.state.internal.InternalMergingState;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.util.RowDataUtil;
+import org.apache.flink.table.runtime.generated.GeneratedNamespaceAggsHandleFunction;
+import org.apache.flink.table.runtime.operators.window.TimeWindow;
+import org.apache.flink.table.runtime.operators.window.groupwindow.internal.InternalWindowProcessFunction;
+import org.apache.flink.table.runtime.operators.window.groupwindow.internal.MergingWindowProcessFunction;
+import org.apache.flink.table.runtime.operators.window.groupwindow.triggers.EventTimeTriggers;
+import org.apache.flink.table.runtime.operators.window.groupwindow.triggers.ProcessingTimeTriggers;
+import org.apache.flink.table.runtime.operators.window.groupwindow.triggers.Trigger;
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowTimerService;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceAssigner;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnslicingWindowProcessor;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnslicingWindowTimerServiceImpl;
+import org.apache.flink.util.Preconditions;
+
+import java.time.ZoneId;
+import java.util.Collection;
+import java.util.Optional;
+
+import static java.util.Objects.requireNonNull;
+import static org.apache.flink.table.runtime.util.TimeWindowUtil.toEpochMillsForTimer;
+
+/**
+ * An window aggregate processor implementation which works for {@link UnsliceAssigner}, e.g.
+ * session windows.
+ */
+public class UnsliceWindowAggProcessor extends AbstractWindowAggProcessor<TimeWindow>
+ implements UnslicingWindowProcessor<TimeWindow> {
+
+ private final UnsliceAssigner<TimeWindow> unsliceAssigner;
+
+ private final Trigger<TimeWindow> trigger;
+
+ // ----------------------------------------------------------------------------------------
+
+ private transient MetricGroup metrics;
+
+ protected transient MergingWindowProcessFunction<RowData, TimeWindow> windowFunction;
+
+ private transient TriggerContextImpl triggerContext;
+
+ public UnsliceWindowAggProcessor(
+ GeneratedNamespaceAggsHandleFunction<TimeWindow> genAggsHandler,
+ UnsliceAssigner<TimeWindow> unsliceAssigner,
+ TypeSerializer<RowData> accSerializer,
+ int indexOfCountStar,
+ ZoneId shiftTimeZone) {
+ super(
+ genAggsHandler,
+ unsliceAssigner,
+ accSerializer,
+ unsliceAssigner.isEventTime(),
+ indexOfCountStar,
+ shiftTimeZone);
+ this.unsliceAssigner = unsliceAssigner;
+ if (isEventTime) {
+ trigger = EventTimeTriggers.afterEndOfWindow();
+ } else {
+ trigger = ProcessingTimeTriggers.afterEndOfWindow();
+ }
+ }
+
+ @Override
+ public void open(Context<TimeWindow> context) throws Exception {
+ super.open(context);
+ this.metrics = context.getRuntimeContext().getMetricGroup();
+ this.windowFunction =
+ new MergingWindowProcessFunction<>(
+ unsliceAssigner.getMergingWindowAssigner(),
+ aggregator,
+ unsliceAssigner
+ .getMergingWindowAssigner()
+ .getWindowSerializer(new ExecutionConfig()),
+ // TODO support allowedLateness
+ 0L);
+
+ triggerContext = new TriggerContextImpl();
+ triggerContext.open();
+
+ WindowContextImpl windowContext = new WindowContextImpl();
+
+ this.windowFunction.open(windowContext);
+ }
+
+ @Override
+ public boolean processElement(RowData key, RowData element) throws Exception {
+ // the windows which the input row should be placed into
+ Optional<TimeWindow> affectedWindowOp =
+ unsliceAssigner.assignStateNamespace(element, clockService, windowFunction);
+ boolean isElementDropped = true;
+ if (affectedWindowOp.isPresent()) {
+ TimeWindow affectedWindow = affectedWindowOp.get();
+ isElementDropped = false;
+
+ RowData acc = windowState.value(affectedWindow);
+ if (acc == null) {
+ acc = aggregator.createAccumulators();
+ }
+ aggregator.setAccumulators(affectedWindow, acc);
+
+ if (RowDataUtil.isAccumulateMsg(element)) {
+ aggregator.accumulate(element);
+ } else {
+ aggregator.retract(element);
+ }
+ acc = aggregator.getAccumulators();
+ windowState.update(affectedWindow, acc);
+ }
+
+ // the actual window which the input row is belongs to
+ Optional<TimeWindow> actualWindowOp =
+ unsliceAssigner.assignActualWindow(element, clockService, windowFunction);
+ Preconditions.checkArgument(
+ (affectedWindowOp.isPresent() && actualWindowOp.isPresent())
+ || (!affectedWindowOp.isPresent() && !actualWindowOp.isPresent()));
+
+ if (actualWindowOp.isPresent()) {
+ TimeWindow actualWindow = actualWindowOp.get();
+ triggerContext.setWindow(actualWindow);
+ // register a timer for the window to fire and clean up
+ long triggerTime = toEpochMillsForTimer(actualWindow.maxTimestamp(), shiftTimeZone);
+ if (isEventTime) {
+ triggerContext.registerEventTimeTimer(triggerTime);
+ } else {
+ triggerContext.registerProcessingTimeTimer(triggerTime);
+ }
+ }
+ return isElementDropped;
+ }
+
+ @Override
+ public void fireWindow(long timerTimestamp, TimeWindow window) throws Exception {
+ windowFunction.prepareAggregateAccumulatorForEmit(window);
+ RowData aggResult = aggregator.getValue(window);
+ triggerContext.setWindow(window);
+ final boolean isFired;
+ if (isEventTime) {
+ isFired = triggerContext.onEventTime(timerTimestamp);
+ } else {
+ isFired = triggerContext.onProcessingTime(timerTimestamp);
+ }
+ // we shouldn't emit an empty window
+ if (isFired && !emptySupplier.get()) {
+ collect(aggResult);
+ }
+ }
+
+ @Override
+ public void clearWindow(long timerTimestamp, TimeWindow window) throws Exception {
+ windowFunction.cleanWindowIfNeeded(window, timerTimestamp);
+ }
+
+ @Override
+ public void advanceProgress(long progress) throws Exception {}
+
+ @Override
+ public void prepareCheckpoint() throws Exception {}
+
+ @Override
+ public TypeSerializer<TimeWindow> createWindowSerializer() {
+ return unsliceAssigner
+ .getMergingWindowAssigner()
+ .getWindowSerializer(new ExecutionConfig());
+ }
+
+ @Override
+ protected WindowTimerService<TimeWindow> getWindowTimerService() {
+ return new UnslicingWindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
+ }
+
+ private class WindowContextImpl
+ implements InternalWindowProcessFunction.Context<RowData, TimeWindow> {
+
+ @Override
+ public long currentProcessingTime() {
+ return ctx.getTimerService().currentProcessingTime();
+ }
+
+ @Override
+ public long currentWatermark() {
+ return ctx.getTimerService().currentWatermark();
+ }
+
+ @Override
+ public ZoneId getShiftTimeZone() {
+ return shiftTimeZone;
+ }
+
+ @Override
+ public RowData getWindowAccumulators(TimeWindow window) throws Exception {
+ return windowState.value(window);
+ }
+
+ @Override
+ public void setWindowAccumulators(TimeWindow window, RowData acc) throws Exception {
+ windowState.update(window, acc);
+ }
+
+ @Override
+ public void clearWindowState(TimeWindow window) throws Exception {
+ windowState.clear(window);
+ aggregator.cleanup(window);
+ }
+
+ @Override
+ public void clearPreviousState(TimeWindow window) throws Exception {}
+
+ @Override
+ public void clearTrigger(TimeWindow window) throws Exception {
+ triggerContext.setWindow(window);
+ triggerContext.clear();
+ }
+
+ @Override
+ public void deleteCleanupTimer(TimeWindow window) throws Exception {
+ long cleanupTime = toEpochMillsForTimer(window.maxTimestamp(), shiftTimeZone);
+ if (cleanupTime == Long.MAX_VALUE) {
+ // no need to clean up because we didn't set one
+ return;
+ }
+ if (unsliceAssigner.isEventTime()) {
+ triggerContext.deleteEventTimeTimer(cleanupTime);
+ } else {
+ triggerContext.deleteProcessingTimeTimer(cleanupTime);
+ }
+ }
+
+ @Override
+ public void onMerge(TimeWindow newWindow, Collection<TimeWindow> mergedWindows)
+ throws Exception {
+ triggerContext.setWindow(newWindow);
+ triggerContext.setMergedWindows(mergedWindows);
+ triggerContext.onMerge();
+ }
+
+ @Override
+ public <S extends State> S getPartitionedState(StateDescriptor<S, ?> stateDescriptor)
+ throws Exception {
+ requireNonNull(stateDescriptor, "The state properties must not be null");
+ return ctx.getKeyedStateBackend()
+ .getPartitionedState(
+ VoidNamespace.INSTANCE,
+ VoidNamespaceSerializer.INSTANCE,
+ stateDescriptor);
+ }
+
+ @Override
+ public RowData currentKey() {
+ return ctx.getKeyedStateBackend().getCurrentKey();
+ }
+ }
+
+ private class TriggerContextImpl implements Trigger.OnMergeContext {
+
+ private TimeWindow window;
+
+ private Collection<TimeWindow> mergedWindows;
+
+ public void open() throws Exception {
+ trigger.open(this);
+ }
+
+ @Override
+ public MetricGroup getMetricGroup() {
+ return metrics;
+ }
+
+ public boolean onProcessingTime(long time) throws Exception {
+ return trigger.onProcessingTime(time, window);
+ }
+
+ public boolean onEventTime(long time) throws Exception {
+ return trigger.onEventTime(time, window);
+ }
+
+ public void onMerge() throws Exception {
+ trigger.onMerge(window, this);
+ }
+
+ public void setWindow(TimeWindow window) {
+ this.window = window;
+ }
+
+ public void setMergedWindows(Collection<TimeWindow> mergedWindows) {
+ this.mergedWindows = mergedWindows;
+ }
+
+ @Override
+ public long getCurrentProcessingTime() {
+ return ctx.getTimerService().currentProcessingTime();
+ }
+
+ @Override
+ public long getCurrentWatermark() {
+ return ctx.getTimerService().currentWatermark();
+ }
+
+ @Override
+ public void registerProcessingTimeTimer(long time) {
+ ctx.getTimerService().registerProcessingTimeTimer(window, time);
+ }
+
+ @Override
+ public void registerEventTimeTimer(long time) {
+ ctx.getTimerService().registerEventTimeTimer(window, time);
+ }
+
+ @Override
+ public void deleteProcessingTimeTimer(long time) {
+ ctx.getTimerService().deleteProcessingTimeTimer(window, time);
+ }
+
+ @Override
+ public void deleteEventTimeTimer(long time) {
+ ctx.getTimerService().deleteEventTimeTimer(window, time);
+ }
+
+ @Override
+ public ZoneId getShiftTimeZone() {
+ return shiftTimeZone;
+ }
+
+ public void clear() throws Exception {
+ trigger.clear(window);
+ }
+
+ @Override
+ public <S extends MergingState<?, ?>> void mergePartitionedState(
+ StateDescriptor<S, ?> stateDescriptor) {
+ if (mergedWindows != null && !mergedWindows.isEmpty()) {
+ try {
+ State state =
+ ctx.getKeyedStateBackend()
+ .getOrCreateKeyedState(
+ createWindowSerializer(), stateDescriptor);
+ if (state instanceof InternalMergingState) {
+ ((InternalMergingState<RowData, TimeWindow, ?, ?, ?>) state)
+ .mergeNamespaces(window, mergedWindows);
+ } else {
+ throw new IllegalArgumentException(
+ "The given state descriptor does not refer to a mergeable state (MergingState)");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Error while merging state.", e);
+ }
+ }
+ }
+
+ @Override
+ public <S extends State> S getPartitionedState(StateDescriptor<S, ?> stateDescriptor) {
+ try {
+ return ctx.getKeyedStateBackend()
+ .getPartitionedState(
+ VoidNamespace.INSTANCE,
+ VoidNamespaceSerializer.INSTANCE,
+ stateDescriptor);
+ } catch (Exception e) {
+ throw new RuntimeException("Could not retrieve state", e);
+ }
+ }
+ }
+}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/deduplicate/window/processors/RowTimeWindowDeduplicateProcessor.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/deduplicate/window/processors/RowTimeWindowDeduplicateProcessor.java
index 2217d2451ad..2898f28e56f 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/deduplicate/window/processors/RowTimeWindowDeduplicateProcessor.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/deduplicate/window/processors/RowTimeWindowDeduplicateProcessor.java
@@ -124,7 +124,7 @@ public final class RowTimeWindowDeduplicateProcessor implements SlicingWindowPro
}
@Override
- public void clearWindow(Long windowEnd) throws Exception {
+ public void clearWindow(long timerTimestamp, Long windowEnd) throws Exception {
windowState.clear(windowEnd);
}
@@ -141,7 +141,7 @@ public final class RowTimeWindowDeduplicateProcessor implements SlicingWindowPro
}
@Override
- public void fireWindow(Long windowEnd) throws Exception {
+ public void fireWindow(long timerTimestamp, Long windowEnd) throws Exception {
RowData data = windowState.value(windowEnd);
if (data != null) {
ctx.output(data);
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/window/processors/WindowRankProcessor.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/window/processors/WindowRankProcessor.java
index b25da6421a5..11ac6d524f0 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/window/processors/WindowRankProcessor.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/rank/window/processors/WindowRankProcessor.java
@@ -171,7 +171,7 @@ public final class WindowRankProcessor implements SlicingWindowProcessor<Long> {
}
@Override
- public void clearWindow(Long windowEnd) throws Exception {
+ public void clearWindow(long timerTimestamp, Long windowEnd) throws Exception {
windowState.clear(windowEnd);
}
@@ -188,7 +188,7 @@ public final class WindowRankProcessor implements SlicingWindowProcessor<Long> {
}
@Override
- public void fireWindow(Long windowEnd) throws Exception {
+ public void fireWindow(long timerTimestamp, Long windowEnd) throws Exception {
TopNBuffer buffer = new TopNBuffer(sortKeyComparator, ArrayList::new);
// step 1: load state data into TopNBuffer
Iterator<Map.Entry<RowData, List<RowData>>> stateIterator = windowState.iterator(windowEnd);
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/window/tvf/common/WindowOperatorBase.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/window/tvf/common/WindowOperatorBase.java
index 7fe7f71e1b4..e4303058312 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/window/tvf/common/WindowOperatorBase.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/window/tvf/common/WindowOperatorBase.java
@@ -246,8 +246,8 @@ public abstract class WindowOperatorBase<K, W> extends TableStreamOperator<RowDa
private void onTimer(InternalTimer<K, W> timer) throws Exception {
setCurrentKey(timer.getKey());
W window = timer.getNamespace();
- windowProcessor.fireWindow(window);
- windowProcessor.clearWindow(window);
+ windowProcessor.fireWindow(timer.getTimestamp(), window);
+ windowProcessor.clearWindow(timer.getTimestamp(), window);
// we don't need to clear window timers,
// because there should only be one timer for each window now, which is current timer.
}
diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/window/tvf/common/WindowProcessor.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/window/tvf/common/WindowProcessor.java
index 1bebb25820c..6f221a47fc1 100644
--- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/window/tvf/common/WindowProcessor.java
+++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/window/tvf/common/WindowProcessor.java
@@ -72,18 +72,20 @@ public interface WindowProcessor<W> extends Serializable {
*
* <p>Note: the key context has been set.
*
+ * @param timerTimestamp the fired timestamp
* @param window the window to emit
*/
- void fireWindow(W window) throws Exception;
+ void fireWindow(long timerTimestamp, W window) throws Exception;
/**
* Clear state and resources associated with the given window namespace.
*
* <p>Note: the key context has been set.
*
+ * @param timerTimestamp the fired timestamp
* @param window the window to clear
*/
- void clearWindow(W window) throws Exception;
+ void clearWindow(long timerTimestamp, W window) throws Exception;
/**
* The tear-down method of the function. It is called after the last call to the main working
diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/SlicingWindowAggOperatorTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/SlicingWindowAggOperatorTest.java
index 6886c1db028..7d199df60fb 100644
--- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/SlicingWindowAggOperatorTest.java
+++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/SlicingWindowAggOperatorTest.java
@@ -18,115 +18,54 @@
package org.apache.flink.table.runtime.operators.aggregate.window;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
-import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.utils.JoinedRowData;
-import org.apache.flink.table.runtime.dataview.StateDataViewStore;
-import org.apache.flink.table.runtime.generated.GeneratedNamespaceAggsHandleFunction;
-import org.apache.flink.table.runtime.generated.NamespaceAggsHandleFunction;
-import org.apache.flink.table.runtime.keyselector.RowDataKeySelector;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigner;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SliceAssigners;
import org.apache.flink.table.runtime.operators.window.tvf.slicing.SlicingWindowOperator;
-import org.apache.flink.table.runtime.typeutils.PagedTypeSerializer;
-import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
-import org.apache.flink.table.runtime.util.GenericRowRecordSortComparator;
-import org.apache.flink.table.runtime.util.RowDataHarnessAssertor;
-import org.apache.flink.table.types.logical.BigIntType;
-import org.apache.flink.table.types.logical.IntType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.table.types.logical.TimestampType;
-import org.apache.flink.table.types.logical.VarCharType;
-import org.apache.flink.table.utils.HandwrittenSelectorUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.time.Duration;
-import java.time.LocalDateTime;
import java.time.ZoneId;
-import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.Collection;
import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.flink.table.data.TimestampData.fromEpochMillis;
import static org.apache.flink.table.runtime.util.StreamRecordUtils.insertRecord;
-import static org.apache.flink.table.runtime.util.TimeWindowUtil.toUtcTimestampMills;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.assertj.core.api.Assertions.fail;
-/** Tests for window aggregate operators created by {@link SlicingWindowAggOperatorBuilder}. */
+/** Tests for slicing window aggregate operators created by {@link WindowAggOperatorBuilder}. */
@RunWith(Parameterized.class)
-public class SlicingWindowAggOperatorTest {
-
- private static final ZoneId UTC_ZONE_ID = ZoneId.of("UTC");
- private static final ZoneId SHANGHAI_ZONE_ID = ZoneId.of("Asia/Shanghai");
- private final ZoneId shiftTimeZone;
+public class SlicingWindowAggOperatorTest extends WindowAggOperatorTestBase {
public SlicingWindowAggOperatorTest(ZoneId shiftTimeZone) {
- this.shiftTimeZone = shiftTimeZone;
+ super(shiftTimeZone);
}
- private static final RowType INPUT_ROW_TYPE =
- new RowType(
- Arrays.asList(
- new RowType.RowField("f0", new VarCharType(Integer.MAX_VALUE)),
- new RowType.RowField("f1", new IntType()),
- new RowType.RowField("f2", new TimestampType())));
-
- private static final RowDataSerializer INPUT_ROW_SER = new RowDataSerializer(INPUT_ROW_TYPE);
-
- private static final RowDataSerializer ACC_SER =
- new RowDataSerializer(new BigIntType(), new BigIntType());
-
- private static final LogicalType[] OUTPUT_TYPES =
- new LogicalType[] {
- new VarCharType(Integer.MAX_VALUE),
- new BigIntType(),
- new BigIntType(),
- new BigIntType(),
- new BigIntType()
- };
-
- private static final RowDataKeySelector KEY_SELECTOR =
- HandwrittenSelectorUtil.getRowDataSelector(
- new int[] {0}, INPUT_ROW_TYPE.getChildren().toArray(new LogicalType[0]));
-
- private static final PagedTypeSerializer<RowData> KEY_SER =
- (PagedTypeSerializer<RowData>) KEY_SELECTOR.getProducedType().toSerializer();
-
- private static final TypeSerializer<RowData> OUT_SERIALIZER =
- new RowDataSerializer(OUTPUT_TYPES);
-
- private static final RowDataHarnessAssertor ASSERTER =
- new RowDataHarnessAssertor(
- OUTPUT_TYPES, new GenericRowRecordSortComparator(0, VarCharType.STRING_TYPE));
-
@Test
public void testEventTimeHoppingWindows() throws Exception {
final SliceAssigner assigner =
SliceAssigners.hopping(
2, shiftTimeZone, Duration.ofSeconds(3), Duration.ofSeconds(1));
- final SumAndCountAggsFunction aggsFunction = new SumAndCountAggsFunction(assigner);
+ final SlicingSumAndCountAggsFunction aggsFunction =
+ new SlicingSumAndCountAggsFunction(assigner);
SlicingWindowOperator<RowData, ?> operator =
- SlicingWindowAggOperatorBuilder.builder()
- .inputSerializer(INPUT_ROW_SER)
- .shiftTimeZone(shiftTimeZone)
- .keySerializer(KEY_SER)
- .assigner(assigner)
- .aggregate(wrapGenerated(aggsFunction), ACC_SER)
- .countStarIndex(1)
- .build();
+ (SlicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .countStarIndex(1)
+ .build();
OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
createTestHarness(operator);
@@ -224,16 +163,18 @@ public class SlicingWindowAggOperatorTest {
public void testProcessingTimeHoppingWindows() throws Exception {
final SliceAssigner assigner =
SliceAssigners.hopping(-1, shiftTimeZone, Duration.ofHours(3), Duration.ofHours(1));
- final SumAndCountAggsFunction aggsFunction = new SumAndCountAggsFunction(assigner);
+ final SlicingSumAndCountAggsFunction aggsFunction =
+ new SlicingSumAndCountAggsFunction(assigner);
SlicingWindowOperator<RowData, ?> operator =
- SlicingWindowAggOperatorBuilder.builder()
- .inputSerializer(INPUT_ROW_SER)
- .shiftTimeZone(shiftTimeZone)
- .keySerializer(KEY_SER)
- .assigner(assigner)
- .aggregate(wrapGenerated(aggsFunction), ACC_SER)
- .countStarIndex(1)
- .build();
+ (SlicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .countStarIndex(1)
+ .build();
OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
createTestHarness(operator);
@@ -346,15 +287,17 @@ public class SlicingWindowAggOperatorTest {
final SliceAssigner assigner =
SliceAssigners.cumulative(
2, shiftTimeZone, Duration.ofSeconds(3), Duration.ofSeconds(1));
- final SumAndCountAggsFunction aggsFunction = new SumAndCountAggsFunction(assigner);
+ final SlicingSumAndCountAggsFunction aggsFunction =
+ new SlicingSumAndCountAggsFunction(assigner);
SlicingWindowOperator<RowData, ?> operator =
- SlicingWindowAggOperatorBuilder.builder()
- .inputSerializer(INPUT_ROW_SER)
- .shiftTimeZone(shiftTimeZone)
- .keySerializer(KEY_SER)
- .assigner(assigner)
- .aggregate(wrapGenerated(aggsFunction), ACC_SER)
- .build();
+ (SlicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .build();
OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
createTestHarness(operator);
@@ -462,15 +405,17 @@ public class SlicingWindowAggOperatorTest {
final SliceAssigner assigner =
SliceAssigners.cumulative(
-1, shiftTimeZone, Duration.ofDays(1), Duration.ofHours(8));
- final SumAndCountAggsFunction aggsFunction = new SumAndCountAggsFunction(assigner);
+ final SlicingSumAndCountAggsFunction aggsFunction =
+ new SlicingSumAndCountAggsFunction(assigner);
SlicingWindowOperator<RowData, ?> operator =
- SlicingWindowAggOperatorBuilder.builder()
- .inputSerializer(INPUT_ROW_SER)
- .shiftTimeZone(shiftTimeZone)
- .keySerializer(KEY_SER)
- .assigner(assigner)
- .aggregate(wrapGenerated(aggsFunction), ACC_SER)
- .build();
+ (SlicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .build();
OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
createTestHarness(operator);
@@ -596,15 +541,17 @@ public class SlicingWindowAggOperatorTest {
public void testEventTimeTumblingWindows() throws Exception {
final SliceAssigner assigner =
SliceAssigners.tumbling(2, shiftTimeZone, Duration.ofSeconds(3));
- final SumAndCountAggsFunction aggsFunction = new SumAndCountAggsFunction(assigner);
+ final SlicingSumAndCountAggsFunction aggsFunction =
+ new SlicingSumAndCountAggsFunction(assigner);
SlicingWindowOperator<RowData, ?> operator =
- SlicingWindowAggOperatorBuilder.builder()
- .inputSerializer(INPUT_ROW_SER)
- .shiftTimeZone(shiftTimeZone)
- .keySerializer(KEY_SER)
- .assigner(assigner)
- .aggregate(wrapGenerated(aggsFunction), ACC_SER)
- .build();
+ (SlicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .build();
OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
createTestHarness(operator);
@@ -703,15 +650,17 @@ public class SlicingWindowAggOperatorTest {
// [1970-01-01 00:00, 1970-01-01 05:00] <=> [1969-12-31 16:00, 1969-12-31 21:00]
// [1970-01-01 05:00, 1970-01-01 10:00] <=> [1969-12-31 21:00, 1970-01-01 02:00]
- final SumAndCountAggsFunction aggsFunction = new SumAndCountAggsFunction(assigner);
+ final SlicingSumAndCountAggsFunction aggsFunction =
+ new SlicingSumAndCountAggsFunction(assigner);
SlicingWindowOperator<RowData, ?> operator =
- SlicingWindowAggOperatorBuilder.builder()
- .inputSerializer(INPUT_ROW_SER)
- .shiftTimeZone(shiftTimeZone)
- .keySerializer(KEY_SER)
- .assigner(assigner)
- .aggregate(wrapGenerated(aggsFunction), ACC_SER)
- .build();
+ (SlicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .build();
OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
createTestHarness(operator);
@@ -777,188 +726,44 @@ public class SlicingWindowAggOperatorTest {
final SliceAssigner assigner =
SliceAssigners.hopping(
2, shiftTimeZone, Duration.ofSeconds(3), Duration.ofSeconds(1));
- final SumAndCountAggsFunction aggsFunction = new SumAndCountAggsFunction(assigner);
+ final SlicingSumAndCountAggsFunction aggsFunction =
+ new SlicingSumAndCountAggsFunction(assigner);
// hopping window without specifying count star index
assertThatThrownBy(
() ->
- SlicingWindowAggOperatorBuilder.builder()
+ WindowAggOperatorBuilder.builder()
.inputSerializer(INPUT_ROW_SER)
.shiftTimeZone(shiftTimeZone)
.keySerializer(KEY_SER)
.assigner(assigner)
- .aggregate(wrapGenerated(aggsFunction), ACC_SER)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
.build())
.hasMessageContaining(
"Hopping window requires a COUNT(*) in the aggregate functions.");
}
- /** Get the timestamp in mills by given epoch mills and timezone. */
- private long localMills(long epochMills) {
- return toUtcTimestampMills(epochMills, shiftTimeZone);
- }
-
- private static OneInputStreamOperatorTestHarness<RowData, RowData> createTestHarness(
- SlicingWindowOperator<RowData, ?> operator) throws Exception {
- return new KeyedOneInputStreamOperatorTestHarness<>(
- operator, KEY_SELECTOR, KEY_SELECTOR.getProducedType());
- }
-
- private static GeneratedNamespaceAggsHandleFunction<Long> wrapGenerated(
- NamespaceAggsHandleFunction<Long> aggsFunction) {
- return new GeneratedNamespaceAggsHandleFunction<Long>("N/A", "", new Object[0]) {
- private static final long serialVersionUID = 1L;
-
- @Override
- public NamespaceAggsHandleFunction<Long> newInstance(ClassLoader classLoader) {
- return aggsFunction;
- }
- };
- }
-
- /**
- * This performs a {@code SUM(f1), COUNT(f1)}, where f1 is BIGINT type. The return value
- * contains {@code sum, count, window_start, window_end}.
- */
- private static class SumAndCountAggsFunction implements NamespaceAggsHandleFunction<Long> {
-
- private static final long serialVersionUID = 1L;
+ /** A test agg function for {@link SlicingWindowAggOperatorTest}. */
+ protected static class SlicingSumAndCountAggsFunction
+ extends SumAndCountAggsFunctionBase<Long> {
private final SliceAssigner assigner;
- boolean openCalled;
- final AtomicInteger closeCalled = new AtomicInteger(0);
-
- long sum;
- boolean sumIsNull;
- long count;
- boolean countIsNull;
-
- protected transient JoinedRowData result;
-
- private SumAndCountAggsFunction(SliceAssigner assigner) {
+ public SlicingSumAndCountAggsFunction(SliceAssigner assigner) {
this.assigner = assigner;
}
- public void open(StateDataViewStore store) throws Exception {
- openCalled = true;
- result = new JoinedRowData();
- }
-
- public void setAccumulators(Long window, RowData acc) throws Exception {
- if (!openCalled) {
- fail("Open was not called");
- }
- sumIsNull = acc.isNullAt(0);
- if (!sumIsNull) {
- sum = acc.getLong(0);
- } else {
- sum = 0L;
- }
-
- countIsNull = acc.isNullAt(1);
- if (!countIsNull) {
- count = acc.getLong(1);
- } else {
- count = 0L;
- }
- }
-
- public void accumulate(RowData inputRow) throws Exception {
- if (!openCalled) {
- fail("Open was not called");
- }
- boolean inputIsNull = inputRow.isNullAt(1);
- if (!inputIsNull) {
- sum += inputRow.getInt(1);
- count += 1;
- sumIsNull = false;
- countIsNull = false;
- }
- }
-
- public void retract(RowData inputRow) throws Exception {
- if (!openCalled) {
- fail("Open was not called");
- }
- boolean inputIsNull = inputRow.isNullAt(1);
- if (!inputIsNull) {
- sum -= inputRow.getInt(1);
- count -= 1;
- }
- }
-
- public void merge(Long window, RowData otherAcc) throws Exception {
- if (!openCalled) {
- fail("Open was not called");
- }
- boolean sumIsNull2 = otherAcc.isNullAt(0);
- if (!sumIsNull2) {
- sum += otherAcc.getLong(0);
- sumIsNull = false;
- }
- boolean countIsNull2 = otherAcc.isNullAt(1);
- if (!countIsNull2) {
- count += otherAcc.getLong(1);
- countIsNull = false;
- }
- }
-
- public RowData createAccumulators() {
- if (!openCalled) {
- fail("Open was not called");
- }
- GenericRowData rowData = new GenericRowData(2);
- rowData.setField(1, 0L); // count has default 0 value
- return rowData;
- }
-
- public RowData getAccumulators() throws Exception {
- if (!openCalled) {
- fail("Open was not called");
- }
- GenericRowData row = new GenericRowData(2);
- if (!sumIsNull) {
- row.setField(0, sum);
- }
- if (!countIsNull) {
- row.setField(1, count);
- }
- return row;
- }
-
- public void cleanup(Long window) {}
-
- public void close() {
- closeCalled.incrementAndGet();
+ @Override
+ protected long getWindowStart(Long window) {
+ return assigner.getWindowStart(window);
}
@Override
- public RowData getValue(Long window) throws Exception {
- if (!openCalled) {
- fail("Open was not called");
- }
- GenericRowData row = new GenericRowData(4);
- if (!sumIsNull) {
- row.setField(0, sum);
- }
- if (!countIsNull) {
- row.setField(1, count);
- }
- row.setField(1, count);
- row.setField(2, assigner.getWindowStart(window));
- row.setField(3, window);
- return row;
+ protected long getWindowEnd(Long window) {
+ return window;
}
}
- /** Get epoch mills from a timestamp string and the time zone the timestamp belongs. */
- private static long epochMills(ZoneId shiftTimeZone, String timestampStr) {
- LocalDateTime localDateTime = LocalDateTime.parse(timestampStr);
- ZoneOffset zoneOffset = shiftTimeZone.getRules().getOffset(localDateTime);
- return localDateTime.toInstant(zoneOffset).toEpochMilli();
- }
-
@Parameterized.Parameters(name = "TimeZone = {0}")
public static Collection<Object[]> runMode() {
return Arrays.asList(new Object[] {UTC_ZONE_ID}, new Object[] {SHANGHAI_ZONE_ID});
diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/UnslicingWindowAggOperatorTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/UnslicingWindowAggOperatorTest.java
new file mode 100644
index 00000000000..4fd499f20a5
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/UnslicingWindowAggOperatorTest.java
@@ -0,0 +1,628 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.aggregate.window;
+
+import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.runtime.keyselector.EmptyRowDataKeySelector;
+import org.apache.flink.table.runtime.operators.window.TimeWindow;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceAssigner;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnsliceAssigners;
+import org.apache.flink.table.runtime.operators.window.tvf.unslicing.UnslicingWindowOperator;
+import org.apache.flink.table.runtime.typeutils.PagedTypeSerializer;
+import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
+import org.apache.flink.table.runtime.util.GenericRowRecordSortComparator;
+import org.apache.flink.table.runtime.util.RowDataHarnessAssertor;
+import org.apache.flink.table.types.logical.BigIntType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.flink.types.RowKind;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.time.Duration;
+import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import static org.apache.flink.table.data.TimestampData.fromEpochMillis;
+import static org.apache.flink.table.runtime.util.StreamRecordUtils.binaryRecord;
+import static org.apache.flink.table.runtime.util.StreamRecordUtils.insertRecord;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for unslicing window aggregate operators created by {@link WindowAggOperatorBuilder}. */
+@RunWith(Parameterized.class)
+public class UnslicingWindowAggOperatorTest extends WindowAggOperatorTestBase {
+
+ public UnslicingWindowAggOperatorTest(ZoneId shiftTimeZone) {
+ super(shiftTimeZone);
+ }
+
+ @Test
+ public void testEventTimeSessionWindows() throws Exception {
+ final UnsliceAssigner<TimeWindow> assigner =
+ UnsliceAssigners.session(2, shiftTimeZone, Duration.ofSeconds(3));
+
+ final UnslicingSumAndCountAggsFunction aggsFunction =
+ new UnslicingSumAndCountAggsFunction();
+ UnslicingWindowOperator<RowData, ?> operator =
+ (UnslicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .build();
+
+ OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
+ createTestHarness(operator);
+
+ testHarness.setup(OUT_SERIALIZER);
+ testHarness.open();
+
+ // process elements
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ // add elements out-of-order
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(3999L)));
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(3000L)));
+
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(20L)));
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(0L)));
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(999L)));
+
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(1998L)));
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(1999L)));
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(1000L)));
+
+ testHarness.processWatermark(new Watermark(999));
+ expectedOutput.add(new Watermark(999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.processWatermark(new Watermark(1999));
+ expectedOutput.add(new Watermark(1999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // do a snapshot, close and restore again
+ testHarness.prepareSnapshotPreBarrier(0L);
+ OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
+ testHarness.close();
+
+ assertThat(aggsFunction.closeCalled.get()).as("Close was not called.").isGreaterThan(0);
+
+ expectedOutput.clear();
+ testHarness = createTestHarness(operator);
+ testHarness.setup();
+ testHarness.initializeState(snapshot);
+ testHarness.open();
+
+ testHarness.processWatermark(new Watermark(2999));
+ expectedOutput.add(new Watermark(2999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.processWatermark(new Watermark(3999));
+ expectedOutput.add(insertRecord("key1", 3L, 3L, localMills(0L), localMills(3999L)));
+ expectedOutput.add(new Watermark(3999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // late element
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(3500L)));
+
+ testHarness.processWatermark(new Watermark(4999));
+ expectedOutput.add(new Watermark(4999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // late for all assigned windows, should be dropped
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(999L)));
+
+ testHarness.processWatermark(new Watermark(5999));
+ expectedOutput.add(new Watermark(5999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.processWatermark(new Watermark(6999));
+ expectedOutput.add(insertRecord("key2", 6L, 6L, localMills(1000L), localMills(6999L)));
+ expectedOutput.add(new Watermark(6999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // those don't have any effect...
+ testHarness.processWatermark(new Watermark(7999));
+ testHarness.processWatermark(new Watermark(8999));
+ expectedOutput.add(new Watermark(7999));
+ expectedOutput.add(new Watermark(8999));
+
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ assertThat(operator.getNumLateRecordsDropped().getCount()).isEqualTo(1);
+
+ testHarness.close();
+ }
+
+ @Test
+ public void testEventTimeSessionWindowsWithChangelog() throws Exception {
+ final UnsliceAssigner<TimeWindow> assigner =
+ UnsliceAssigners.session(2, shiftTimeZone, Duration.ofSeconds(3));
+
+ final UnslicingSumAndCountAggsFunction aggsFunction =
+ new UnslicingSumAndCountAggsFunction();
+ UnslicingWindowOperator<RowData, ?> operator =
+ (UnslicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .countStarIndex(1)
+ .build();
+
+ OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
+ createTestHarness(operator);
+
+ testHarness.setup(OUT_SERIALIZER);
+ testHarness.open();
+
+ // process elements
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ // add elements out-of-order
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(3999L)));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(3000L)));
+
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key1", 1, fromEpochMillis(20L)));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key1", 1, fromEpochMillis(0L)));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key1", 1, fromEpochMillis(999L)));
+
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(1998L)));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(1999L)));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(1000L)));
+
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key3", 1, fromEpochMillis(1999L)));
+
+ testHarness.processElement(binaryRecord(RowKind.DELETE, "key4", 1, fromEpochMillis(999L)));
+
+ testHarness.processWatermark(new Watermark(999));
+ expectedOutput.add(new Watermark(999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.processElement(binaryRecord(RowKind.DELETE, "key3", 1, fromEpochMillis(2900L)));
+ testHarness.processWatermark(new Watermark(1999));
+ expectedOutput.add(new Watermark(1999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // do a snapshot, close and restore again
+ testHarness.prepareSnapshotPreBarrier(0L);
+ OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
+ testHarness.close();
+
+ assertThat(aggsFunction.closeCalled.get()).as("Close was not called.").isGreaterThan(0);
+
+ expectedOutput.clear();
+ testHarness = createTestHarness(operator);
+ testHarness.setup();
+ testHarness.initializeState(snapshot);
+ testHarness.open();
+
+ testHarness.processWatermark(new Watermark(2999));
+ expectedOutput.add(new Watermark(2999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // TODO split the session if necessary when receiving -D after fixing FLINK-34039
+ // session window does not re-split correctly as expected when consuming -D, only acc is
+ // retracted
+ testHarness.processElement(binaryRecord(RowKind.DELETE, "key2", 1, fromEpochMillis(3999L)));
+ testHarness.processElement(binaryRecord(RowKind.DELETE, "key2", 1, fromEpochMillis(3000L)));
+
+ testHarness.processWatermark(new Watermark(3999));
+ expectedOutput.add(
+ binaryRecord(RowKind.INSERT, "key1", 3L, 3L, localMills(0L), localMills(3999L)));
+ // TODO align the behavior when receiving single -D within a window after fixing FLINK-33760
+ expectedOutput.add(
+ binaryRecord(
+ RowKind.INSERT, "key4", null, -1L, localMills(999L), localMills(3999L)));
+ expectedOutput.add(new Watermark(3999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // late element
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(3500L)));
+ testHarness.processElement(
+ binaryRecord(RowKind.UPDATE_BEFORE, "key2", 1, fromEpochMillis(3000L)));
+ testHarness.processElement(
+ binaryRecord(RowKind.UPDATE_AFTER, "key2", 1, fromEpochMillis(3800L)));
+
+ testHarness.processWatermark(new Watermark(4999));
+ expectedOutput.add(new Watermark(4999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // late for all assigned windows, should be dropped
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key1", 1, fromEpochMillis(999L)));
+
+ testHarness.processWatermark(new Watermark(5999));
+ expectedOutput.add(new Watermark(5999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.processWatermark(new Watermark(6999));
+ expectedOutput.add(
+ binaryRecord(RowKind.INSERT, "key2", 4L, 4L, localMills(1000L), localMills(6999L)));
+ expectedOutput.add(new Watermark(6999));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // those don't have any effect...
+ testHarness.processWatermark(new Watermark(7999));
+ testHarness.processWatermark(new Watermark(8999));
+ expectedOutput.add(new Watermark(7999));
+ expectedOutput.add(new Watermark(8999));
+
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ assertThat(operator.getNumLateRecordsDropped().getCount()).isEqualTo(1);
+
+ testHarness.close();
+ }
+
+ @Test
+ public void testProcessingTimeSessionWindows() throws Exception {
+ final UnsliceAssigner<TimeWindow> assigner =
+ UnsliceAssigners.session(-1, shiftTimeZone, Duration.ofSeconds(3));
+
+ final UnslicingSumAndCountAggsFunction aggsFunction =
+ new UnslicingSumAndCountAggsFunction();
+ UnslicingWindowOperator<RowData, ?> operator =
+ (UnslicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .build();
+
+ OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
+ createTestHarness(operator);
+
+ testHarness.setup(OUT_SERIALIZER);
+ testHarness.open();
+
+ // process elements
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:03"));
+
+ // timestamp is ignored in processing time
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(Long.MAX_VALUE)));
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(7000L)));
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(7000L)));
+
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(7000L)));
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(7000L)));
+
+ // do a snapshot, close and restore again
+ testHarness.prepareSnapshotPreBarrier(0L);
+ OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
+ testHarness.close();
+
+ assertThat(aggsFunction.closeCalled.get()).as("Close was not called.").isGreaterThan(0);
+
+ testHarness = createTestHarness(operator);
+ testHarness.setup();
+ testHarness.initializeState(snapshot);
+ testHarness.open();
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:06"));
+
+ expectedOutput.add(
+ insertRecord(
+ "key2",
+ 3L,
+ 3L,
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:03"),
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:06")));
+ expectedOutput.add(
+ insertRecord(
+ "key1",
+ 2L,
+ 2L,
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:03"),
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:06")));
+
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:07"));
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(7000L)));
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:08"));
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(7000L)));
+
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:12"));
+
+ expectedOutput.add(
+ insertRecord(
+ "key1",
+ 2L,
+ 2L,
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:07"),
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:11")));
+
+ assertThat(operator.getWatermarkLatency().getValue()).isEqualTo(Long.valueOf(0L));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+ testHarness.close();
+ }
+
+ @Test
+ public void testProcessingTimeSessionWindowsWithChangelog() throws Exception {
+ final UnsliceAssigner<TimeWindow> assigner =
+ UnsliceAssigners.session(-1, shiftTimeZone, Duration.ofSeconds(3));
+
+ final UnslicingSumAndCountAggsFunction aggsFunction =
+ new UnslicingSumAndCountAggsFunction();
+ UnslicingWindowOperator<RowData, ?> operator =
+ (UnslicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(KEY_SER)
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .countStarIndex(1)
+ .build();
+
+ OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
+ createTestHarness(operator);
+
+ testHarness.setup(OUT_SERIALIZER);
+ testHarness.open();
+
+ // process elements
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:03"));
+
+ // timestamp is ignored in processing time
+ testHarness.processElement(
+ binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(Long.MAX_VALUE)));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(7000L)));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key2", 1, fromEpochMillis(7000L)));
+
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key1", 1, fromEpochMillis(7000L)));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key1", 1, fromEpochMillis(7000L)));
+
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key3", 1, fromEpochMillis(7000L)));
+ testHarness.processElement(binaryRecord(RowKind.DELETE, "key3", 1, fromEpochMillis(7000L)));
+
+ testHarness.processElement(binaryRecord(RowKind.DELETE, "key4", 1, fromEpochMillis(7000L)));
+
+ // do a snapshot, close and restore again
+ testHarness.prepareSnapshotPreBarrier(0L);
+ OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
+ testHarness.close();
+
+ assertThat(aggsFunction.closeCalled.get()).as("Close was not called.").isGreaterThan(0);
+
+ testHarness = createTestHarness(operator);
+ testHarness.setup();
+ testHarness.initializeState(snapshot);
+ testHarness.open();
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:06"));
+
+ expectedOutput.add(
+ binaryRecord(
+ RowKind.INSERT,
+ "key2",
+ 3L,
+ 3L,
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:03"),
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:06")));
+ expectedOutput.add(
+ binaryRecord(
+ RowKind.INSERT,
+ "key1",
+ 2L,
+ 2L,
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:03"),
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:06")));
+
+ // TODO align the behavior when receiving single -D within a window after fixing FLINK-33760
+ expectedOutput.add(
+ binaryRecord(
+ RowKind.INSERT,
+ "key4",
+ null,
+ -1L,
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:03"),
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:06")));
+
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:07"));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key1", 1, fromEpochMillis(7000L)));
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:08"));
+ testHarness.processElement(binaryRecord(RowKind.INSERT, "key1", 1, fromEpochMillis(8000L)));
+
+ // TODO split the session if necessary when receiving -D after fixing FLINK-34039
+ // session window does not re-split correctly as expected when consuming -D, only acc is
+ // retracted
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:10"));
+ testHarness.processElement(
+ binaryRecord(RowKind.UPDATE_BEFORE, "key1", 1, fromEpochMillis(8000L)));
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:11"));
+ testHarness.processElement(
+ binaryRecord(RowKind.UPDATE_AFTER, "key1", 1, fromEpochMillis(6000L)));
+
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:12"));
+
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:13"));
+
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.setProcessingTime(epochMills(shiftTimeZone, "1970-01-01T00:00:14"));
+
+ expectedOutput.add(
+ binaryRecord(
+ RowKind.INSERT,
+ "key1",
+ 2L,
+ 2L,
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:07"),
+ epochMills(UTC_ZONE_ID, "1970-01-01T00:00:14")));
+
+ assertThat(operator.getWatermarkLatency().getValue()).isEqualTo(Long.valueOf(0L));
+ ASSERTER.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+ testHarness.close();
+ }
+
+ @Test
+ public void testSessionWindowsWithoutPartitionKey() throws Exception {
+ // there is no key (type string) in the output
+ final LogicalType[] outputTypes =
+ new LogicalType[] {
+ new BigIntType(), new BigIntType(), new BigIntType(), new BigIntType()
+ };
+ final RowDataHarnessAssertor asserter =
+ new RowDataHarnessAssertor(
+ outputTypes,
+ new GenericRowRecordSortComparator(0, VarCharType.STRING_TYPE));
+
+ final UnsliceAssigner<TimeWindow> assigner =
+ UnsliceAssigners.session(2, shiftTimeZone, Duration.ofSeconds(3));
+
+ final EmptyRowDataKeySelector keySelector = EmptyRowDataKeySelector.INSTANCE;
+ final UnslicingSumAndCountAggsFunction aggsFunction =
+ new UnslicingSumAndCountAggsFunction();
+ UnslicingWindowOperator<RowData, ?> operator =
+ (UnslicingWindowOperator<RowData, ?>)
+ WindowAggOperatorBuilder.builder()
+ .inputSerializer(INPUT_ROW_SER)
+ .shiftTimeZone(shiftTimeZone)
+ .keySerializer(
+ (PagedTypeSerializer<RowData>)
+ keySelector.getProducedType().toSerializer())
+ .assigner(assigner)
+ .aggregate(createGeneratedAggsHandle(aggsFunction), ACC_SER)
+ .build();
+
+ OneInputStreamOperatorTestHarness<RowData, RowData> testHarness =
+ new KeyedOneInputStreamOperatorTestHarness<>(
+ operator, keySelector, keySelector.getProducedType());
+
+ testHarness.setup(new RowDataSerializer(outputTypes));
+ testHarness.open();
+
+ // process elements
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+
+ // add elements with multi keys
+ testHarness.processElement(insertRecord("key2", 1, fromEpochMillis(3999L)));
+
+ testHarness.processElement(insertRecord("key1", 1, fromEpochMillis(20L)));
+
+ testHarness.processWatermark(new Watermark(999));
+ expectedOutput.add(new Watermark(999));
+ asserter.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.processElement(insertRecord("key3", 1, fromEpochMillis(2990L)));
+
+ testHarness.processWatermark(new Watermark(1999));
+ expectedOutput.add(new Watermark(1999));
+ asserter.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ // do a snapshot, close and restore again
+ testHarness.prepareSnapshotPreBarrier(0L);
+ OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
+ testHarness.close();
+
+ assertThat(aggsFunction.closeCalled.get()).as("Close was not called.").isGreaterThan(0);
+
+ expectedOutput.clear();
+ testHarness =
+ new KeyedOneInputStreamOperatorTestHarness<>(
+ operator, keySelector, keySelector.getProducedType());
+ testHarness.setup();
+ testHarness.initializeState(snapshot);
+ testHarness.open();
+
+ testHarness.processWatermark(new Watermark(6999));
+ expectedOutput.add(insertRecord(3L, 3L, localMills(20L), localMills(6999L)));
+ expectedOutput.add(new Watermark(6999));
+ asserter.assertOutputEqualsSorted(
+ "Output was not correct.", expectedOutput, testHarness.getOutput());
+
+ testHarness.close();
+ }
+
+ /** A test agg function for {@link UnslicingWindowAggOperatorTest}. */
+ protected static class UnslicingSumAndCountAggsFunction
+ extends SumAndCountAggsFunctionBase<TimeWindow> {
+
+ @Override
+ protected long getWindowStart(TimeWindow window) {
+ return window.getStart();
+ }
+
+ @Override
+ protected long getWindowEnd(TimeWindow window) {
+ return window.getEnd();
+ }
+ }
+
+ @Parameterized.Parameters(name = "TimeZone = {0}")
+ public static Collection<Object[]> runMode() {
+ return Arrays.asList(new Object[] {UTC_ZONE_ID}, new Object[] {SHANGHAI_ZONE_ID});
+ }
+}
diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/WindowAggOperatorTestBase.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/WindowAggOperatorTestBase.java
new file mode 100644
index 00000000000..88c1aabc0ec
--- /dev/null
+++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/aggregate/window/WindowAggOperatorTestBase.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.aggregate.window;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.StateDataViewStore;
+import org.apache.flink.table.runtime.generated.GeneratedNamespaceAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.NamespaceAggsHandleFunction;
+import org.apache.flink.table.runtime.keyselector.RowDataKeySelector;
+import org.apache.flink.table.runtime.operators.window.tvf.common.WindowOperatorBase;
+import org.apache.flink.table.runtime.typeutils.PagedTypeSerializer;
+import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
+import org.apache.flink.table.runtime.util.GenericRowRecordSortComparator;
+import org.apache.flink.table.runtime.util.RowDataHarnessAssertor;
+import org.apache.flink.table.types.logical.BigIntType;
+import org.apache.flink.table.types.logical.IntType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.flink.table.utils.HandwrittenSelectorUtil;
+
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.flink.table.runtime.util.TimeWindowUtil.toUtcTimestampMills;
+import static org.assertj.core.api.Assertions.fail;
+
+/** A test base for window aggregate operator. */
+public abstract class WindowAggOperatorTestBase {
+
+ protected static final ZoneId UTC_ZONE_ID = ZoneId.of("UTC");
+ protected static final ZoneId SHANGHAI_ZONE_ID = ZoneId.of("Asia/Shanghai");
+
+ protected final ZoneId shiftTimeZone;
+
+ public WindowAggOperatorTestBase(ZoneId shiftTimeZone) {
+ this.shiftTimeZone = shiftTimeZone;
+ }
+
+ /** Get the timestamp in mills by given epoch mills and timezone. */
+ protected long localMills(long epochMills) {
+ return toUtcTimestampMills(epochMills, shiftTimeZone);
+ }
+
+ // ============================== Utils ==============================
+
+ // ============================== Util Fields ==============================
+
+ private static final RowType INPUT_ROW_TYPE =
+ new RowType(
+ Arrays.asList(
+ new RowType.RowField("f0", new VarCharType(Integer.MAX_VALUE)),
+ new RowType.RowField("f1", new IntType()),
+ new RowType.RowField("f2", new TimestampType())));
+
+ protected static final RowDataSerializer INPUT_ROW_SER = new RowDataSerializer(INPUT_ROW_TYPE);
+
+ protected static final RowDataSerializer ACC_SER =
+ new RowDataSerializer(new BigIntType(), new BigIntType());
+
+ protected static final LogicalType[] OUTPUT_TYPES =
+ new LogicalType[] {
+ new VarCharType(Integer.MAX_VALUE),
+ new BigIntType(),
+ new BigIntType(),
+ new BigIntType(),
+ new BigIntType()
+ };
+
+ protected static final RowDataKeySelector KEY_SELECTOR =
+ HandwrittenSelectorUtil.getRowDataSelector(
+ new int[] {0}, INPUT_ROW_TYPE.getChildren().toArray(new LogicalType[0]));
+
+ protected static final PagedTypeSerializer<RowData> KEY_SER =
+ (PagedTypeSerializer<RowData>) KEY_SELECTOR.getProducedType().toSerializer();
+
+ protected static final TypeSerializer<RowData> OUT_SERIALIZER =
+ new RowDataSerializer(OUTPUT_TYPES);
+
+ protected static final RowDataHarnessAssertor ASSERTER =
+ new RowDataHarnessAssertor(
+ OUTPUT_TYPES, new GenericRowRecordSortComparator(0, VarCharType.STRING_TYPE));
+
+ // ============================== Util Functions ==============================
+
+ protected static OneInputStreamOperatorTestHarness<RowData, RowData> createTestHarness(
+ WindowOperatorBase<RowData, ?> operator) throws Exception {
+ return new KeyedOneInputStreamOperatorTestHarness<>(
+ operator, KEY_SELECTOR, KEY_SELECTOR.getProducedType());
+ }
+
+ protected static <T> GeneratedNamespaceAggsHandleFunction<T> createGeneratedAggsHandle(
+ NamespaceAggsHandleFunction<T> aggsFunction) {
+ return new GeneratedNamespaceAggsHandleFunction<T>("N/A", "", new Object[0]) {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public NamespaceAggsHandleFunction<T> newInstance(ClassLoader classLoader) {
+ return aggsFunction;
+ }
+ };
+ }
+
+ /** Get epoch mills from a timestamp string and the time zone the timestamp belongs. */
+ protected static long epochMills(ZoneId shiftTimeZone, String timestampStr) {
+ LocalDateTime localDateTime = LocalDateTime.parse(timestampStr);
+ ZoneOffset zoneOffset = shiftTimeZone.getRules().getOffset(localDateTime);
+ return localDateTime.toInstant(zoneOffset).toEpochMilli();
+ }
+
+ /**
+ * This performs a {@code SUM(f1), COUNT(f1)}, where f1 is BIGINT type. The return value
+ * contains {@code sum, count, window_start, window_end}.
+ */
+ protected abstract static class SumAndCountAggsFunctionBase<T>
+ implements NamespaceAggsHandleFunction<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ boolean openCalled;
+ final AtomicInteger closeCalled = new AtomicInteger(0);
+
+ long sum;
+ boolean sumIsNull;
+ long count;
+ boolean countIsNull;
+
+ protected transient JoinedRowData result;
+
+ public void open(StateDataViewStore store) throws Exception {
+ openCalled = true;
+ result = new JoinedRowData();
+ }
+
+ public void setAccumulators(T window, RowData acc) throws Exception {
+ if (!openCalled) {
+ fail("Open was not called");
+ }
+ sumIsNull = acc.isNullAt(0);
+ if (!sumIsNull) {
+ sum = acc.getLong(0);
+ } else {
+ sum = 0L;
+ }
+
+ countIsNull = acc.isNullAt(1);
+ if (!countIsNull) {
+ count = acc.getLong(1);
+ } else {
+ count = 0L;
+ }
+ }
+
+ public void accumulate(RowData inputRow) throws Exception {
+ if (!openCalled) {
+ fail("Open was not called");
+ }
+ boolean inputIsNull = inputRow.isNullAt(1);
+ if (!inputIsNull) {
+ sum += inputRow.getInt(1);
+ count += 1;
+ sumIsNull = false;
+ countIsNull = false;
+ }
+ }
+
+ public void retract(RowData inputRow) throws Exception {
+ if (!openCalled) {
+ fail("Open was not called");
+ }
+ boolean inputIsNull = inputRow.isNullAt(1);
+ if (!inputIsNull) {
+ sum -= inputRow.getInt(1);
+ count -= 1;
+ }
+ }
+
+ public void merge(T window, RowData otherAcc) throws Exception {
+ if (!openCalled) {
+ fail("Open was not called");
+ }
+ boolean sumIsNull2 = otherAcc.isNullAt(0);
+ if (!sumIsNull2) {
+ sum += otherAcc.getLong(0);
+ sumIsNull = false;
+ }
+ boolean countIsNull2 = otherAcc.isNullAt(1);
+ if (!countIsNull2) {
+ count += otherAcc.getLong(1);
+ countIsNull = false;
+ }
+ }
+
+ public RowData createAccumulators() {
+ if (!openCalled) {
+ fail("Open was not called");
+ }
+ GenericRowData rowData = new GenericRowData(2);
+ rowData.setField(1, 0L); // count has default 0 value
+ return rowData;
+ }
+
+ public RowData getAccumulators() throws Exception {
+ if (!openCalled) {
+ fail("Open was not called");
+ }
+ GenericRowData row = new GenericRowData(2);
+ if (!sumIsNull) {
+ row.setField(0, sum);
+ } else {
+ row.setField(0, null);
+ }
+ if (!countIsNull) {
+ row.setField(1, count);
+ } else {
+ row.setField(1, null);
+ }
+ return row;
+ }
+
+ public void cleanup(T window) {}
+
+ public void close() {
+ closeCalled.incrementAndGet();
+ }
+
+ @Override
+ public RowData getValue(T window) throws Exception {
+ if (!openCalled) {
+ fail("Open was not called");
+ }
+ GenericRowData row = new GenericRowData(4);
+ if (!sumIsNull) {
+ row.setField(0, sum);
+ } else {
+ row.setField(0, null);
+ }
+ if (!countIsNull) {
+ row.setField(1, count);
+ } else {
+ row.setField(1, null);
+ }
+
+ row.setField(2, getWindowStart(window));
+ row.setField(3, getWindowEnd(window));
+ return row;
+ }
+
+ protected abstract long getWindowStart(T window);
+
+ protected abstract long getWindowEnd(T window);
+ }
+}