You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2016/06/21 10:17:18 UTC
[1/6] flink git commit: [FLINK-3714] Add Support for "Allowed
Lateness"
Repository: flink
Updated Branches:
refs/heads/master f2e9c521a -> 09e28109b
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
index 94b0e3c..ddf4dcc 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
@@ -28,8 +28,6 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.typeutils.TypeInfoParser;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.streaming.api.datastream.WindowedStream;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -47,6 +45,8 @@ import org.apache.flink.streaming.api.windowing.triggers.ContinuousEventTimeTrig
import org.apache.flink.streaming.api.windowing.triggers.CountTrigger;
import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
import org.apache.flink.streaming.api.windowing.triggers.PurgingTrigger;
+import org.apache.flink.streaming.api.windowing.triggers.Trigger;
+import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.streaming.api.windowing.windows.Window;
@@ -60,8 +60,6 @@ import org.apache.flink.util.Collector;
import org.junit.Assert;
import org.junit.Test;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -166,7 +164,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
operator.setInputType(inputType, new ExecutionConfig());
@@ -203,7 +202,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
operator.setInputType(inputType, new ExecutionConfig());
@@ -306,7 +306,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
@@ -341,7 +342,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
@@ -379,7 +381,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
@@ -455,7 +458,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
@@ -533,7 +537,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
- PurgingTrigger.of(CountTrigger.of(4)));
+ PurgingTrigger.of(CountTrigger.of(4)),
+ 0);
operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
@@ -638,7 +643,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new SessionWindowFunction()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
@@ -700,7 +706,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()),
- ContinuousEventTimeTrigger.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)));
+ ContinuousEventTimeTrigger.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+ 0);
operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
@@ -753,7 +760,9 @@ public class WindowOperatorTest {
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
testHarness.processWatermark(new Watermark(initialTime + 6000));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE));
+
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE));
+
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), Long.MAX_VALUE));
expectedOutput.add(new Watermark(6000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
@@ -790,7 +799,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()),
- PurgingTrigger.of(CountTrigger.of(WINDOW_SIZE)));
+ PurgingTrigger.of(CountTrigger.of(WINDOW_SIZE)),
+ 0);
operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse(
"Tuple2<String, Integer>"), new ExecutionConfig());
@@ -865,7 +875,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
@@ -900,7 +911,8 @@ public class WindowOperatorTest {
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
- EventTimeTrigger.create());
+ EventTimeTrigger.create(),
+ 0);
OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> otherTestHarness =
new OneInputStreamOperatorTestHarness<>(otherOperator);
@@ -917,6 +929,780 @@ public class WindowOperatorTest {
Assert.assertEquals(operator.processingTimeTimerTimestamps, otherOperator.processingTimeTimerTimestamps);
}
+ @Test
+ public void testLateness() throws Exception {
+ final int WINDOW_SIZE = 2;
+ final long LATENESS = 500;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
+ new WindowOperator<>(
+ TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+ EventTimeTrigger.create(),
+ LATENESS);
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ operator.setInputType(inputType, new ExecutionConfig());
+ testHarness.open();
+
+ long initialTime = 0L;
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 500));
+ testHarness.processWatermark(new Watermark(initialTime + 1500));
+
+ expected.add(new Watermark(initialTime + 1500));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1300));
+ testHarness.processWatermark(new Watermark(initialTime + 2300));
+
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 1999));
+ expected.add(new Watermark(initialTime + 2300));
+
+ // this will not be dropped because window.maxTimestamp() + allowedLateness > currentWatermark
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1997));
+ testHarness.processWatermark(new Watermark(initialTime + 6000));
+
+ // this is 1 and not 3 because the trigger fires and purges
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
+ expected.add(new Watermark(initialTime + 6000));
+
+ // this will be dropped because window.maxTimestamp() + allowedLateness < currentWatermark
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
+ testHarness.processWatermark(new Watermark(initialTime + 7000));
+
+ expected.add(new Watermark(initialTime + 7000));
+
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testCleanupTimeOverflow() throws Exception {
+ final int WINDOW_SIZE = 1;
+ final long LATENESS = 2000;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
+ new WindowOperator<>(
+ TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+ EventTimeTrigger.create(),
+ LATENESS);
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ operator.setInputType(inputType, new ExecutionConfig());
+ testHarness.open();
+
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ long timestamp = Long.MAX_VALUE - 1750;
+ long windowSize = WINDOW_SIZE * 1000;
+ long endOfWindow = (timestamp - (timestamp % windowSize)) + windowSize - 1;
+
+ // normal element
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), Long.MAX_VALUE - 1750));
+ testHarness.processWatermark(new Watermark(Long.MAX_VALUE - 1500));
+ testHarness.processWatermark(new Watermark(endOfWindow));
+
+ // guarantee that the cleanup would overflow
+ Assert.assertTrue(endOfWindow + LATENESS < endOfWindow);
+
+ // guarantee that the cleanup would have happened if it were not for the overflow check
+ Assert.assertTrue(endOfWindow + LATENESS < Long.MAX_VALUE - 1500);
+ Assert.assertTrue(Long.MAX_VALUE - 1500 < endOfWindow);
+ Assert.assertTrue(endOfWindow < Long.MAX_VALUE);
+
+ expected.add(new Watermark(Long.MAX_VALUE - 1500));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), endOfWindow));
+ expected.add(new Watermark(endOfWindow));
+
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testDropDueToLatenessTumbling() throws Exception {
+ final int WINDOW_SIZE = 2;
+ final long LATENESS = 0;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
+ new WindowOperator<>(
+ TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+ EventTimeTrigger.create(),
+ LATENESS);
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ operator.setInputType(inputType, new ExecutionConfig());
+ testHarness.open();
+
+ long initialTime = 0L;
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ // normal element
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processWatermark(new Watermark(initialTime + 1985));
+
+ expected.add(new Watermark(initialTime + 1985));
+
+ // a late that is added to an active window
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1980));
+ testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 1999));
+ expected.add(new Watermark(initialTime + 1999));
+
+ // dropped as late
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2001));
+ testHarness.processWatermark(new Watermark(initialTime + 2999));
+
+ expected.add(new Watermark(initialTime + 2999));
+
+ testHarness.processWatermark(new Watermark(initialTime + 3999));
+
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));
+ expected.add(new Watermark(initialTime + 3999));
+
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testDropDueToLatenessSliding() throws Exception {
+ final int WINDOW_SIZE = 3;
+ final int WINDOW_SLIDE = 1;
+ final long LATENESS = 0;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
+ new WindowOperator<>(
+ SlidingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS), Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
+ EventTimeTrigger.create(),
+ LATENESS);
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ operator.setInputType(inputType, new ExecutionConfig());
+ testHarness.open();
+
+ long initialTime = 0L;
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
+ expected.add(new Watermark(initialTime + 1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
+ testHarness.processWatermark(new Watermark(initialTime + 3000));
+
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 2999));
+ expected.add(new Watermark(initialTime + 3000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 3001));
+
+ // lateness is set to 0 and window_size = 3 sec and slide 1, the following 2 elements (2400) are late,
+ // and assigned to windows ending at 2999, 3999, 4999.
+ // The 2999 is dropped because it is already inactive (WM = 2999) but the rest are kept.
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 3001));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3900));
+ testHarness.processWatermark(new Watermark(initialTime + 6000));
+
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), initialTime + 3999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 3999));
+
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), initialTime + 4999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 4999));
+
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 5999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 5999));
+
+ expected.add(new Watermark(initialTime + 6000));
+
+ // dropped due to lateness
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 3001));
+
+ testHarness.processWatermark(new Watermark(initialTime + 25000));
+
+ expected.add(new Watermark(initialTime + 25000));
+
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testDropDueToLatenessSessionZeroLateness() throws Exception {
+ final int GAP_SIZE = 3;
+ final long LATENESS = 0;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
+ new WindowOperator<>(
+ EventTimeSessionWindows.withGap(Time.seconds(GAP_SIZE)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
+ EventTimeTrigger.create(),
+ LATENESS);
+
+ operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ testHarness.open();
+
+ long initialTime = 0L;
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ expected.add(new Watermark(1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
+ testHarness.processWatermark(new Watermark(initialTime + 4998));
+
+ expected.add(new Watermark(4998));
+
+ // late but added
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
+ testHarness.processWatermark(new Watermark(initialTime + 7400));
+
+ expected.add(new Watermark(7400));
+
+ // late element to merge the previous sessions
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
+ testHarness.processWatermark(new Watermark(initialTime + 11501));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new Watermark(11501));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
+ testHarness.processWatermark(new Watermark(initialTime + 14600));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new Watermark(14600));
+
+ // NON-ACCUM TRIGGER
+ // with 0 lateness this is dropped because it is assigned to a non-active window, (testDropDueToLatenessSessionZeroLateness)
+ // with a small lateness (e.g. 10), this is assigned to window 10000-13000 (the 11600 is GCed after firing) (testDropDueToLatenessSessionWithLateness)
+ // finally with a big lateness (e.g. 10000) the first window is not GCed so this merges all the windows so far (testDropDueToLatenessSessionWithHugeLateness)
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
+ testHarness.processWatermark(new Watermark(initialTime + 20000));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500l, 17500l), 17499));
+ expected.add(new Watermark(20000));
+
+ testHarness.processWatermark(new Watermark(initialTime + 100000));
+
+ expected.add(new Watermark(100000));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testDropDueToLatenessSessionZeroLatenessAccum() throws Exception {
+
+ final int GAP_SIZE = 3;
+ final long LATENESS = 0;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
+ new WindowOperator<>(
+ EventTimeSessionWindows.withGap(Time.seconds(GAP_SIZE)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
+ new EventTimeTriggerAccum(),
+ LATENESS);
+
+ operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ testHarness.open();
+
+ long initialTime = 0L;
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ expected.add(new Watermark(1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
+ testHarness.processWatermark(new Watermark(initialTime + 4998));
+
+ expected.add(new Watermark(4998));
+
+ // late but added
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
+ testHarness.processWatermark(new Watermark(initialTime + 7400));
+
+ expected.add(new Watermark(7400));
+
+ // late element to merge the previous sessions
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
+ testHarness.processWatermark(new Watermark(initialTime + 11501));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new Watermark(11501));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
+ testHarness.processWatermark(new Watermark(initialTime + 14600));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new Watermark(14600));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
+ testHarness.processWatermark(new Watermark(initialTime + 20000));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500l, 17500l), 17499));
+ expected.add(new Watermark(20000));
+
+ testHarness.processWatermark(new Watermark(initialTime + 100000));
+ expected.add(new Watermark(100000));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testDropDueToLatenessSessionWithLateness() throws Exception {
+ final int GAP_SIZE = 3;
+ final long LATENESS = 10;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
+ new WindowOperator<>(
+ EventTimeSessionWindows.withGap(Time.seconds(GAP_SIZE)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
+ EventTimeTrigger.create(),
+ LATENESS);
+
+ operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ testHarness.open();
+
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+ long initialTime = 0L;
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ expected.add(new Watermark(1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
+ testHarness.processWatermark(new Watermark(initialTime + 4998));
+
+ expected.add(new Watermark(4998));
+
+ // late but added
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
+ testHarness.processWatermark(new Watermark(initialTime + 7400));
+
+ expected.add(new Watermark(7400));
+
+ // late element to merge the previous sessions
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
+ testHarness.processWatermark(new Watermark(initialTime + 11501));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new Watermark(11501));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
+ testHarness.processWatermark(new Watermark(initialTime + 14600));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new Watermark(14600));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
+ testHarness.processWatermark(new Watermark(initialTime + 20000));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500l, 17500l), 17499));
+ expected.add(new Watermark(20000));
+
+ testHarness.processWatermark(new Watermark(initialTime + 100000));
+ expected.add(new Watermark(100000));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testDropDueToLatenessSessionWithLatenessAccum() throws Exception {
+ final int GAP_SIZE = 3;
+ final long LATENESS = 10;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
+ new WindowOperator<>(
+ EventTimeSessionWindows.withGap(Time.seconds(GAP_SIZE)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
+ new EventTimeTriggerAccum(),
+ LATENESS);
+
+ operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ testHarness.open();
+
+ long initialTime = 0L;
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ expected.add(new Watermark(1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
+ testHarness.processWatermark(new Watermark(initialTime + 4998));
+
+ expected.add(new Watermark(4998));
+
+ // late but added
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
+ testHarness.processWatermark(new Watermark(initialTime + 7400));
+
+ expected.add(new Watermark(7400));
+
+ // late element to merge the previous sessions
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
+ testHarness.processWatermark(new Watermark(initialTime + 11501));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new Watermark(11501));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
+ testHarness.processWatermark(new Watermark(initialTime + 14600));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new Watermark(14600));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
+ testHarness.processWatermark(new Watermark(initialTime + 20000));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-2", 10000l, 14600l), 14599));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-3", 10000l, 17500l), 17499));
+ expected.add(new Watermark(20000));
+
+ testHarness.processWatermark(new Watermark(initialTime + 100000));
+
+ expected.add(new Watermark(100000));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testDropDueToLatenessSessionWithHugeLateness() throws Exception {
+
+ final int GAP_SIZE = 3;
+ final long LATENESS = 10000;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
+ new WindowOperator<>(
+ EventTimeSessionWindows.withGap(Time.seconds(GAP_SIZE)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
+ EventTimeTrigger.create(),
+ LATENESS);
+
+ operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ testHarness.open();
+
+ long initialTime = 0L;
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ expected.add(new Watermark(1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
+ testHarness.processWatermark(new Watermark(initialTime + 4998));
+
+ expected.add(new Watermark(4998));
+
+ // late but added
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
+ testHarness.processWatermark(new Watermark(initialTime + 7400));
+
+ expected.add(new Watermark(7400));
+
+ // late element to merge the previous sessions
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
+ testHarness.processWatermark(new Watermark(initialTime + 11501));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new Watermark(11501));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
+ testHarness.processWatermark(new Watermark(initialTime + 14600));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new Watermark(14600));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
+ testHarness.processWatermark(new Watermark(initialTime + 20000));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 10000l, 13000l), 12999));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500l, 17500l), 17499));
+ expected.add(new Watermark(20000));
+
+ testHarness.processWatermark(new Watermark(initialTime + 100000));
+
+ expected.add(new Watermark(100000));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
+ testHarness.close();
+ }
+
+ @Test
+ public void testDropDueToLatenessSessionWithHugeLatenessAccum() throws Exception {
+ final int GAP_SIZE = 3;
+ final long LATENESS = 10000;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents",
+ new SumReducer(),
+ inputType.createSerializer(new ExecutionConfig()));
+
+ WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator =
+ new WindowOperator<>(
+ EventTimeSessionWindows.withGap(Time.seconds(GAP_SIZE)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalSingleValueWindowFunction<>(new ReducedSessionWindowFunction()),
+ new EventTimeTriggerAccum(),
+ LATENESS);
+
+ operator.setInputType(TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"), new ExecutionConfig());
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ testHarness.open();
+
+ long initialTime = 0L;
+ ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ expected.add(new Watermark(1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
+ testHarness.processWatermark(new Watermark(initialTime + 4998));
+
+ expected.add(new Watermark(4998));
+
+ // late but added
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
+ testHarness.processWatermark(new Watermark(initialTime + 7400));
+
+ expected.add(new Watermark(7400));
+
+ // late element to merge the previous sessions
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
+ testHarness.processWatermark(new Watermark(initialTime + 11501));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new Watermark(11501));
+
+ // new session
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
+ testHarness.processWatermark(new Watermark(initialTime + 14600));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new Watermark(14600));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
+ testHarness.processWatermark(new Watermark(initialTime + 20000));
+
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-7", 1000l, 14600l), 14599));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-8", 1000l, 17500l), 17499));
+ expected.add(new Watermark(20000));
+
+ testHarness.processWatermark(new Watermark(initialTime + 100000));
+ expected.add(new Watermark(100000));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
+ testHarness.close();
+ }
+
// ------------------------------------------------------------------------
// UDFs
// ------------------------------------------------------------------------
@@ -1077,4 +1863,57 @@ public class WindowOperatorTest {
return Collections.singletonList(new TimeWindow(timestamp, timestamp + sessionTimeout));
}
}
+
+ /**
+ * A trigger that fires at the end of the window but does not
+ * purge the state of the fired window. This is to test the state
+ * garbage collection mechanism.
+ */
+ public class EventTimeTriggerAccum extends Trigger<Object, TimeWindow> {
+ private static final long serialVersionUID = 1L;
+
+ private EventTimeTriggerAccum() {}
+
+ @Override
+ public TriggerResult onElement(Object element, long timestamp, TimeWindow window, TriggerContext ctx) throws Exception {
+ ctx.registerEventTimeTimer(window.maxTimestamp());
+ return (window.maxTimestamp() <= ctx.getCurrentWatermark()) ?
+ TriggerResult.FIRE :
+ TriggerResult.CONTINUE;
+ }
+
+ @Override
+ public TriggerResult onEventTime(long time, TimeWindow window, TriggerContext ctx) {
+ return time == window.maxTimestamp() ?
+ TriggerResult.FIRE :
+ TriggerResult.CONTINUE;
+ }
+
+ @Override
+ public TriggerResult onProcessingTime(long time, TimeWindow window, TriggerContext ctx) throws Exception {
+ return TriggerResult.CONTINUE;
+ }
+
+ @Override
+ public void clear(TimeWindow window, TriggerContext ctx) throws Exception {
+ ctx.deleteEventTimeTimer(window.maxTimestamp());
+ }
+
+ @Override
+ public boolean canMerge() {
+ return true;
+ }
+
+ @Override
+ public TriggerResult onMerge(TimeWindow window,
+ OnMergeContext ctx) {
+ ctx.registerEventTimeTimer(window.maxTimestamp());
+ return TriggerResult.CONTINUE;
+ }
+
+ @Override
+ public String toString() {
+ return "EventTimeTrigger()";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala
index 020c619..47c13c9 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/AllWindowedStream.scala
@@ -27,6 +27,7 @@ import org.apache.flink.streaming.api.functions.aggregation.{ComparableAggregato
import org.apache.flink.streaming.api.scala.function.AllWindowFunction
import org.apache.flink.streaming.api.scala.function.util.{ScalaAllWindowFunction, ScalaAllWindowFunctionWrapper, ScalaReduceFunction, ScalaFoldFunction}
import org.apache.flink.streaming.api.windowing.evictors.Evictor
+import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.streaming.api.windowing.triggers.Trigger
import org.apache.flink.streaming.api.windowing.windows.Window
import org.apache.flink.util.Collector
@@ -54,7 +55,21 @@ import org.apache.flink.util.Collector
*/
@Public
class AllWindowedStream[T, W <: Window](javaStream: JavaAllWStream[T, W]) {
-
+
+ /**
+ * Sets the allowed lateness to a user-specified value.
+ * If not explicitly set, the allowed lateness is [[Long.MaxValue]].
+ * Setting the allowed lateness is only valid for event-time windows.
+ * If a value different than 0 is provided with a processing-time
+ * [[org.apache.flink.streaming.api.windowing.assigners.WindowAssigner]],
+ * then an exception is thrown.
+ */
+ @PublicEvolving
+ def allowedLateness(lateness: Time): AllWindowedStream[T, W] = {
+ javaStream.allowedLateness(lateness)
+ this
+ }
+
/**
* Sets the [[Trigger]] that should be used to trigger window emission.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala
----------------------------------------------------------------------
diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala
index 773829e..6a10ff6 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedStream.scala
@@ -27,6 +27,7 @@ import org.apache.flink.streaming.api.functions.aggregation.{ComparableAggregato
import org.apache.flink.streaming.api.scala.function.WindowFunction
import org.apache.flink.streaming.api.scala.function.util.{ScalaFoldFunction, ScalaReduceFunction, ScalaWindowFunction, ScalaWindowFunctionWrapper}
import org.apache.flink.streaming.api.windowing.evictors.Evictor
+import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.streaming.api.windowing.triggers.Trigger
import org.apache.flink.streaming.api.windowing.windows.Window
import org.apache.flink.util.Collector
@@ -59,6 +60,20 @@ import org.apache.flink.util.Collector
class WindowedStream[T, K, W <: Window](javaStream: JavaWStream[T, K, W]) {
/**
+ * Sets the allowed lateness to a user-specified value.
+ * If not explicitly set, the allowed lateness is [[Long.MaxValue]].
+ * Setting the allowed lateness is only valid for event-time windows.
+ * If a value different than 0 is provided with a processing-time
+ * [[org.apache.flink.streaming.api.windowing.assigners.WindowAssigner]],
+ * then an exception is thrown.
+ */
+ @PublicEvolving
+ def allowedLateness(lateness: Time): WindowedStream[T, K, W] = {
+ javaStream.allowedLateness(lateness)
+ this
+ }
+
+ /**
* Sets the [[Trigger]] that should be used to trigger window emission.
*/
@PublicEvolving
[6/6] flink git commit: [hotfix] Remove "initialTime" in
WindowOperatorTest
Posted by al...@apache.org.
[hotfix] Remove "initialTime" in WindowOperatorTest
I added this once with the though of varying the start time for window
tests but it just makes stuff harder to parse for now.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/09e28109
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/09e28109
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/09e28109
Branch: refs/heads/master
Commit: 09e28109b3707f23230ea50526b7db8aa166df15
Parents: 87195a8
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Jun 20 16:44:44 2016 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Tue Jun 21 12:16:59 2016 +0200
----------------------------------------------------------------------
.../operators/windowing/WindowOperatorTest.java | 594 +++++++++----------
1 file changed, 288 insertions(+), 306 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/09e28109/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
index f606e66..9aaf683 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
@@ -77,38 +77,36 @@ public class WindowOperatorTest {
private void testSlidingEventTimeWindows(OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness) throws Exception {
- long initialTime = 0L;
-
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
// add elements out-of-order
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
- testHarness.processWatermark(new Watermark(initialTime + 999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), initialTime + 999));
+ testHarness.processWatermark(new Watermark(999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 999));
expectedOutput.add(new Watermark(999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 1999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), initialTime + 1999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), initialTime + 1999));
+ testHarness.processWatermark(new Watermark(1999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 1999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 1999));
expectedOutput.add(new Watermark(1999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 2999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), initialTime + 2999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), initialTime + 2999));
+ testHarness.processWatermark(new Watermark(2999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 2999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
expectedOutput.add(new Watermark(2999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
@@ -119,25 +117,25 @@ public class WindowOperatorTest {
testHarness.restore(snapshot, 10L);
testHarness.open();
- testHarness.processWatermark(new Watermark(initialTime + 3999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), initialTime + 3999));
+ testHarness.processWatermark(new Watermark(3999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
expectedOutput.add(new Watermark(3999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 4999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 4999));
+ testHarness.processWatermark(new Watermark(4999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 4999));
expectedOutput.add(new Watermark(4999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 5999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 5999));
+ testHarness.processWatermark(new Watermark(5999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
expectedOutput.add(new Watermark(5999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
// those don't have any effect...
- testHarness.processWatermark(new Watermark(initialTime + 6999));
- testHarness.processWatermark(new Watermark(initialTime + 7999));
+ testHarness.processWatermark(new Watermark(6999));
+ testHarness.processWatermark(new Watermark(7999));
expectedOutput.add(new Watermark(6999));
expectedOutput.add(new Watermark(7999));
@@ -224,30 +222,29 @@ public class WindowOperatorTest {
}
private void testTumblingEventTimeWindows(OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness) throws Exception {
- long initialTime = 0L;
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
// add elements out-of-order
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
- testHarness.processWatermark(new Watermark(initialTime + 999));
+ testHarness.processWatermark(new Watermark(999));
expectedOutput.add(new Watermark(999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+ testHarness.processWatermark(new Watermark(1999));
expectedOutput.add(new Watermark(1999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
@@ -258,29 +255,29 @@ public class WindowOperatorTest {
testHarness.restore(snapshot, 10L);
testHarness.open();
- testHarness.processWatermark(new Watermark(initialTime + 2999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), initialTime + 2999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), initialTime + 2999));
+ testHarness.processWatermark(new Watermark(2999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), 2999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 3), 2999));
expectedOutput.add(new Watermark(2999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 3999));
+ testHarness.processWatermark(new Watermark(3999));
expectedOutput.add(new Watermark(3999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 4999));
+ testHarness.processWatermark(new Watermark(4999));
expectedOutput.add(new Watermark(4999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 5999));
- expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 5999));
+ testHarness.processWatermark(new Watermark(5999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 5999));
expectedOutput.add(new Watermark(5999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
// those don't have any effect...
- testHarness.processWatermark(new Watermark(initialTime + 6999));
- testHarness.processWatermark(new Watermark(initialTime + 7999));
+ testHarness.processWatermark(new Watermark(6999));
+ testHarness.processWatermark(new Watermark(7999));
expectedOutput.add(new Watermark(6999));
expectedOutput.add(new Watermark(7999));
@@ -392,18 +389,17 @@ public class WindowOperatorTest {
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
- long initialTime = 0L;
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
// add elements out-of-order
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 0));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 1000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), initialTime + 2500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000));
// do a snapshot, close and restore again
StreamTaskState snapshot = testHarness.snapshot(0L, 0L);
@@ -412,28 +408,28 @@ public class WindowOperatorTest {
testHarness.restore(snapshot, 10L);
testHarness.open();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), initialTime + 2500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), initialTime + 5501));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), initialTime + 6000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), initialTime + 6000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), initialTime + 6050));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 5501));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), 6050));
- testHarness.processWatermark(new Watermark(initialTime + 12000));
+ testHarness.processWatermark(new Watermark(12000));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), initialTime + 5499));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), initialTime + 5499));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), 5499));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), initialTime + 9049));
- expectedOutput.add(new Watermark(initialTime + 12000));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), 9049));
+ expectedOutput.add(new Watermark(12000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), initialTime + 15000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), initialTime + 15000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), 15000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), 15000));
- testHarness.processWatermark(new Watermark(initialTime + 17999));
+ testHarness.processWatermark(new Watermark(17999));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), initialTime + 17999));
- expectedOutput.add(new Watermark(initialTime + 17999));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), 17999));
+ expectedOutput.add(new Watermark(17999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
@@ -469,15 +465,14 @@ public class WindowOperatorTest {
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
- long initialTime = 0L;
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
// add elements out-of-order
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 0));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 1000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), initialTime + 2500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500));
// do a snapshot, close and restore again
StreamTaskState snapshot = testHarness.snapshot(0L, 0L);
@@ -486,29 +481,29 @@ public class WindowOperatorTest {
testHarness.restore(snapshot, 10L);
testHarness.open();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 1000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), initialTime + 2500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), initialTime + 5501));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), initialTime + 6000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), initialTime + 6000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), initialTime + 6050));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 5501));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), 6050));
- testHarness.processWatermark(new Watermark(initialTime + 12000));
+ testHarness.processWatermark(new Watermark(12000));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), initialTime + 5499));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), initialTime + 5499));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), initialTime + 9049));
- expectedOutput.add(new Watermark(initialTime + 12000));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), 5499));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), 9049));
+ expectedOutput.add(new Watermark(12000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), initialTime + 15000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), initialTime + 15000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), 15000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), 15000));
- testHarness.processWatermark(new Watermark(initialTime + 17999));
+ testHarness.processWatermark(new Watermark(17999));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), initialTime + 17999));
- expectedOutput.add(new Watermark(initialTime + 17999));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), 17999));
+ expectedOutput.add(new Watermark(17999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
@@ -547,20 +542,19 @@ public class WindowOperatorTest {
new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
// add elements out-of-order
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 0));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 1000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), initialTime + 2500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), initialTime + 3500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 3500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000));
// do a snapshot, close and restore again
StreamTaskState snapshot = testHarness.snapshot(0L, 0L);
@@ -569,21 +563,21 @@ public class WindowOperatorTest {
testHarness.restore(snapshot, 10L);
testHarness.open();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), initialTime + 2500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 6000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 6500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), initialTime + 7000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 6000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 6500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 7000));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-10", 0L, 6500L), initialTime + 6499));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-10", 0L, 6500L), 6499));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
// add an element that merges the two "key1" sessions, they should now have count 6, and therfore fire
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 10), initialTime + 4500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 10), 4500));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-22", 10L, 10000L), initialTime + 9999L));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-22", 10L, 10000L), 9999L));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
@@ -653,15 +647,14 @@ public class WindowOperatorTest {
new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
// add elements out-of-order
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 0));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 33), initialTime + 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 33), 1000));
// do a snapshot, close and restore again
StreamTaskState snapshot = testHarness.snapshot(0L, 0L);
@@ -670,17 +663,17 @@ public class WindowOperatorTest {
testHarness.restore(snapshot, 10L);
testHarness.open();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 33), initialTime + 2500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 33), 2500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 1000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 33), initialTime + 2500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 33), 2500));
- testHarness.processWatermark(new Watermark(initialTime + 12000));
+ testHarness.processWatermark(new Watermark(12000));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-36", 10L, 4000L), initialTime + 3999));
- expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-67", 0L, 3000L), initialTime + 2999));
- expectedOutput.add(new Watermark(initialTime + 12000));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-36", 10L, 4000L), 3999));
+ expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-67", 0L, 3000L), 2999));
+ expectedOutput.add(new Watermark(12000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
@@ -716,51 +709,50 @@ public class WindowOperatorTest {
new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
// The global window actually ignores these timestamps...
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
// add elements out-of-order
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1000));
+ testHarness.processWatermark(new Watermark(1000));
expectedOutput.add(new Watermark(1000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 2000));
+ testHarness.processWatermark(new Watermark(2000));
expectedOutput.add(new Watermark(2000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 3000));
+ testHarness.processWatermark(new Watermark(3000));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE));
expectedOutput.add(new Watermark(3000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 4000));
+ testHarness.processWatermark(new Watermark(4000));
expectedOutput.add(new Watermark(4000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 5000));
+ testHarness.processWatermark(new Watermark(5000));
expectedOutput.add(new Watermark(5000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 6000));
+ testHarness.processWatermark(new Watermark(6000));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 3), Long.MAX_VALUE));
@@ -770,8 +762,8 @@ public class WindowOperatorTest {
// those don't have any effect...
- testHarness.processWatermark(new Watermark(initialTime + 7000));
- testHarness.processWatermark(new Watermark(initialTime + 8000));
+ testHarness.processWatermark(new Watermark(7000));
+ testHarness.processWatermark(new Watermark(8000));
expectedOutput.add(new Watermark(7000));
expectedOutput.add(new Watermark(8000));
@@ -810,8 +802,7 @@ public class WindowOperatorTest {
new OneInputStreamOperatorTestHarness<>(operator);
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
testHarness.open();
@@ -819,15 +810,15 @@ public class WindowOperatorTest {
// The global window actually ignores these timestamps...
// add elements out-of-order
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 20));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 0));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
// do a snapshot, close and restore again
StreamTaskState snapshot = testHarness.snapshot(0L, 0L);
@@ -836,18 +827,18 @@ public class WindowOperatorTest {
testHarness.restore(snapshot, 10L);
testHarness.open();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple2ResultSortComparator());
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE));
expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE));
@@ -959,34 +950,33 @@ public class WindowOperatorTest {
operator.setInputType(inputType, new ExecutionConfig());
testHarness.open();
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 500));
- testHarness.processWatermark(new Watermark(initialTime + 1500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 500));
+ testHarness.processWatermark(new Watermark(1500));
- expected.add(new Watermark(initialTime + 1500));
+ expected.add(new Watermark(1500));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1300));
- testHarness.processWatermark(new Watermark(initialTime + 2300));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1300));
+ testHarness.processWatermark(new Watermark(2300));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 1999));
- expected.add(new Watermark(initialTime + 2300));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 1999));
+ expected.add(new Watermark(2300));
// this will not be dropped because window.maxTimestamp() + allowedLateness > currentWatermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1997));
- testHarness.processWatermark(new Watermark(initialTime + 6000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1997));
+ testHarness.processWatermark(new Watermark(6000));
// this is 1 and not 3 because the trigger fires and purges
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
- expected.add(new Watermark(initialTime + 6000));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
+ expected.add(new Watermark(6000));
// this will be dropped because window.maxTimestamp() + allowedLateness < currentWatermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
- testHarness.processWatermark(new Watermark(initialTime + 7000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
+ testHarness.processWatermark(new Watermark(7000));
- expected.add(new Watermark(initialTime + 7000));
+ expected.add(new Watermark(7000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
testHarness.close();
@@ -1088,34 +1078,33 @@ public class WindowOperatorTest {
operator.setInputType(inputType, new ExecutionConfig());
testHarness.open();
- long initialTime = 0L;
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
// normal element
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1985));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processWatermark(new Watermark(1985));
- expected.add(new Watermark(initialTime + 1985));
+ expected.add(new Watermark(1985));
// this will not be dropped because window.maxTimestamp() + allowedLateness > currentWatermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1980));
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1980));
+ testHarness.processWatermark(new Watermark(1999));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 1999));
- expected.add(new Watermark(initialTime + 1999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 1999));
+ expected.add(new Watermark(1999));
// dropped as late
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1998));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2001));
- testHarness.processWatermark(new Watermark(initialTime + 2999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2001));
+ testHarness.processWatermark(new Watermark(2999));
- expected.add(new Watermark(initialTime + 2999));
+ expected.add(new Watermark(2999));
- testHarness.processWatermark(new Watermark(initialTime + 3999));
+ testHarness.processWatermark(new Watermark(3999));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));
- expected.add(new Watermark(initialTime + 3999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999));
+ expected.add(new Watermark(3999));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
testHarness.close();
@@ -1151,51 +1140,50 @@ public class WindowOperatorTest {
operator.setInputType(inputType, new ExecutionConfig());
testHarness.open();
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processWatermark(new Watermark(1999));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
- expected.add(new Watermark(initialTime + 1999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 1999));
+ expected.add(new Watermark(1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
- testHarness.processWatermark(new Watermark(initialTime + 3000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
+ testHarness.processWatermark(new Watermark(3000));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), initialTime + 2999));
- expected.add(new Watermark(initialTime + 3000));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 2), 2999));
+ expected.add(new Watermark(3000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 3001));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
// lateness is set to 0 and window_size = 3 sec and slide 1, the following 2 elements (2400)
// are assigned to windows ending at 2999, 3999, 4999.
// The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2400));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2400));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 3001));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3900));
- testHarness.processWatermark(new Watermark(initialTime + 6000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 3900));
+ testHarness.processWatermark(new Watermark(6000));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), initialTime + 3999));
- expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 3999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 5), 3999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 3999));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), initialTime + 4999));
- expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 4999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 4), 4999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 4999));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 5999));
- expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), initialTime + 5999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 5999));
+ expected.add(new StreamRecord<>(new Tuple2<>("key1", 2), 5999));
- expected.add(new Watermark(initialTime + 6000));
+ expected.add(new Watermark(6000));
// dropped due to lateness
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 3001));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 3001));
- testHarness.processWatermark(new Watermark(initialTime + 25000));
+ testHarness.processWatermark(new Watermark(25000));
- expected.add(new Watermark(initialTime + 25000));
+ expected.add(new Watermark(25000));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
testHarness.close();
@@ -1231,54 +1219,53 @@ public class WindowOperatorTest {
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.open();
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processWatermark(new Watermark(1999));
expected.add(new Watermark(1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
- testHarness.processWatermark(new Watermark(initialTime + 4998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
+ testHarness.processWatermark(new Watermark(4998));
expected.add(new Watermark(4998));
// this will not be dropped because the session we're adding two has maxTimestamp
// after the current watermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
- testHarness.processWatermark(new Watermark(initialTime + 7400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
+ testHarness.processWatermark(new Watermark(7400));
expected.add(new Watermark(7400));
// this will merge the two sessions into one
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
- testHarness.processWatermark(new Watermark(initialTime + 11501));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+ testHarness.processWatermark(new Watermark(11501));
expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
- testHarness.processWatermark(new Watermark(initialTime + 14600));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
+ testHarness.processWatermark(new Watermark(14600));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
// this is dropped as late
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
- testHarness.processWatermark(new Watermark(initialTime + 20000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
+ testHarness.processWatermark(new Watermark(20000));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500l, 17500l), 17499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
expected.add(new Watermark(20000));
- testHarness.processWatermark(new Watermark(initialTime + 100000));
+ testHarness.processWatermark(new Watermark(100000));
expected.add(new Watermark(100000));
@@ -1324,54 +1311,53 @@ public class WindowOperatorTest {
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.open();
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processWatermark(new Watermark(1999));
expected.add(new Watermark(1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
- testHarness.processWatermark(new Watermark(initialTime + 4998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
+ testHarness.processWatermark(new Watermark(4998));
expected.add(new Watermark(4998));
// this will not be dropped because the session we're adding two has maxTimestamp
// after the current watermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
- testHarness.processWatermark(new Watermark(initialTime + 7400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
+ testHarness.processWatermark(new Watermark(7400));
expected.add(new Watermark(7400));
// this will merge the two sessions into one
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
- testHarness.processWatermark(new Watermark(initialTime + 11501));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+ testHarness.processWatermark(new Watermark(11501));
expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
- testHarness.processWatermark(new Watermark(initialTime + 14600));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
+ testHarness.processWatermark(new Watermark(14600));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
// this is dropped as late
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
- testHarness.processWatermark(new Watermark(initialTime + 20000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
+ testHarness.processWatermark(new Watermark(20000));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
expected.add(new Watermark(20000));
- testHarness.processWatermark(new Watermark(initialTime + 100000));
+ testHarness.processWatermark(new Watermark(100000));
expected.add(new Watermark(100000));
ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
@@ -1415,52 +1401,51 @@ public class WindowOperatorTest {
testHarness.open();
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
- long initialTime = 0L;
-
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processWatermark(new Watermark(1999));
expected.add(new Watermark(1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
- testHarness.processWatermark(new Watermark(initialTime + 4998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
+ testHarness.processWatermark(new Watermark(4998));
expected.add(new Watermark(4998));
// this will not be dropped because the session we're adding two has maxTimestamp
// after the current watermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
- testHarness.processWatermark(new Watermark(initialTime + 7400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
+ testHarness.processWatermark(new Watermark(7400));
expected.add(new Watermark(7400));
// this will merge the two sessions into one
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
- testHarness.processWatermark(new Watermark(initialTime + 11501));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+ testHarness.processWatermark(new Watermark(11501));
expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
- testHarness.processWatermark(new Watermark(initialTime + 14600));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
+ testHarness.processWatermark(new Watermark(14600));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
// dropped as late
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
- testHarness.processWatermark(new Watermark(initialTime + 20000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
+ testHarness.processWatermark(new Watermark(20000));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
expected.add(new Watermark(20000));
- testHarness.processWatermark(new Watermark(initialTime + 100000));
+ testHarness.processWatermark(new Watermark(100000));
expected.add(new Watermark(100000));
ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
@@ -1502,40 +1487,39 @@ public class WindowOperatorTest {
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.open();
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processWatermark(new Watermark(1999));
expected.add(new Watermark(1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
- testHarness.processWatermark(new Watermark(initialTime + 4998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
+ testHarness.processWatermark(new Watermark(4998));
expected.add(new Watermark(4998));
// this will not be dropped because the session we're adding two has maxTimestamp
// after the current watermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
- testHarness.processWatermark(new Watermark(initialTime + 7400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
+ testHarness.processWatermark(new Watermark(7400));
expected.add(new Watermark(7400));
// this will merge the two sessions into one
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
- testHarness.processWatermark(new Watermark(initialTime + 11501));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+ testHarness.processWatermark(new Watermark(11501));
expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
- testHarness.processWatermark(new Watermark(initialTime + 14600));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
+ testHarness.processWatermark(new Watermark(14600));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
@@ -1543,9 +1527,9 @@ public class WindowOperatorTest {
// because of the small allowed lateness and because the trigger is accumulating
// this will be merged into the session (11600-14600) and therefore will not
// be dropped as late
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
// adding ("key2", 1) extended the session to (10000-146000) for which
// maxTimestamp <= currentWatermark. Therefore, we immediately get a firing
@@ -1555,12 +1539,12 @@ public class WindowOperatorTest {
ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
- testHarness.processWatermark(new Watermark(initialTime + 20000));
+ testHarness.processWatermark(new Watermark(20000));
expected.add(new StreamRecord<>(new Tuple3<>("key2-3", 10000L, 17500L), 17499));
expected.add(new Watermark(20000));
- testHarness.processWatermark(new Watermark(initialTime + 100000));
+ testHarness.processWatermark(new Watermark(100000));
expected.add(new Watermark(100000));
@@ -1600,58 +1584,57 @@ public class WindowOperatorTest {
testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
testHarness.open();
-
- long initialTime = 0L;
+
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processWatermark(new Watermark(1999));
expected.add(new Watermark(1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
- testHarness.processWatermark(new Watermark(initialTime + 4998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
+ testHarness.processWatermark(new Watermark(4998));
expected.add(new Watermark(4998));
// this will not be dropped because the session we're adding two has maxTimestamp
// after the current watermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
- testHarness.processWatermark(new Watermark(initialTime + 7400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
+ testHarness.processWatermark(new Watermark(7400));
expected.add(new Watermark(7400));
// this will merge the two sessions into one
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
- testHarness.processWatermark(new Watermark(initialTime + 11501));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+ testHarness.processWatermark(new Watermark(11501));
expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
- testHarness.processWatermark(new Watermark(initialTime + 14600));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
+ testHarness.processWatermark(new Watermark(14600));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 10000L, 13000L), 12999));
ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
- testHarness.processWatermark(new Watermark(initialTime + 20000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
+ testHarness.processWatermark(new Watermark(20000));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
expected.add(new Watermark(20000));
- testHarness.processWatermark(new Watermark(initialTime + 100000));
+ testHarness.processWatermark(new Watermark(100000));
expected.add(new Watermark(100000));
@@ -1691,44 +1674,43 @@ public class WindowOperatorTest {
testHarness.open();
- long initialTime = 0L;
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
- testHarness.processWatermark(new Watermark(initialTime + 1999));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000));
+ testHarness.processWatermark(new Watermark(1999));
expected.add(new Watermark(1999));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2000));
- testHarness.processWatermark(new Watermark(initialTime + 4998));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 2000));
+ testHarness.processWatermark(new Watermark(4998));
expected.add(new Watermark(4998));
// this will not be dropped because the session we're adding two has maxTimestamp
// after the current watermark
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 4500));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 8500));
- testHarness.processWatermark(new Watermark(initialTime + 7400));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 8500));
+ testHarness.processWatermark(new Watermark(7400));
expected.add(new Watermark(7400));
// this will merge the two sessions into one
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
- testHarness.processWatermark(new Watermark(initialTime + 11501));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 7000));
+ testHarness.processWatermark(new Watermark(11501));
expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
- testHarness.processWatermark(new Watermark(initialTime + 14600));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 11600));
+ testHarness.processWatermark(new Watermark(14600));
expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 10000));
// the maxTimestamp of the merged session is already late,
// so we get an immediate firing
@@ -1737,13 +1719,13 @@ public class WindowOperatorTest {
ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
- testHarness.processWatermark(new Watermark(initialTime + 20000));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 14500));
+ testHarness.processWatermark(new Watermark(20000));
expected.add(new StreamRecord<>(new Tuple3<>("key2-8", 1000L, 17500L), 17499));
expected.add(new Watermark(20000));
- testHarness.processWatermark(new Watermark(initialTime + 100000));
+ testHarness.processWatermark(new Watermark(100000));
expected.add(new Watermark(100000));
actual = testHarness.getOutput();
[5/6] flink git commit: [FLINK-3714] Extend comments in
WindowOperatorTest
Posted by al...@apache.org.
[FLINK-3714] Extend comments in WindowOperatorTest
These will make it easier to read the tests for our future selves and
others.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/87195a89
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/87195a89
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/87195a89
Branch: refs/heads/master
Commit: 87195a89ef7bfd6637eff0375bac6ded8a3ca9dc
Parents: 62c5a3c
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Jun 20 16:40:54 2016 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Tue Jun 21 12:16:32 2016 +0200
----------------------------------------------------------------------
.../operators/windowing/WindowOperatorTest.java | 177 ++++++++++++-------
1 file changed, 114 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/87195a89/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
index ddf4dcc..f606e66 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperatorTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.flink.streaming.runtime.operators.windowing;
+import com.google.common.collect.Iterables;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.state.ListStateDescriptor;
@@ -993,7 +994,7 @@ public class WindowOperatorTest {
@Test
public void testCleanupTimeOverflow() throws Exception {
- final int WINDOW_SIZE = 1;
+ final int WINDOW_SIZE = 1000;
final long LATENESS = 2000;
TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
@@ -1002,9 +1003,11 @@ public class WindowOperatorTest {
new SumReducer(),
inputType.createSerializer(new ExecutionConfig()));
+ TumblingEventTimeWindows windowAssigner = TumblingEventTimeWindows.of(Time.milliseconds(WINDOW_SIZE));
+
WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator =
new WindowOperator<>(
- TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+ windowAssigner,
new TimeWindow.Serializer(),
new TupleKeySelector(),
BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
@@ -1024,25 +1027,32 @@ public class WindowOperatorTest {
ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>();
long timestamp = Long.MAX_VALUE - 1750;
- long windowSize = WINDOW_SIZE * 1000;
- long endOfWindow = (timestamp - (timestamp % windowSize)) + windowSize - 1;
+ Collection<TimeWindow> windows = windowAssigner.assignWindows(new Tuple2<>("key2", 1), timestamp);
+ TimeWindow window = Iterables.getOnlyElement(windows);
- // normal element
- testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), Long.MAX_VALUE - 1750));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), timestamp));
+
+ // the garbage collection timer would wrap-around
+ Assert.assertTrue(window.maxTimestamp() + LATENESS < window.maxTimestamp());
+
+ // and it would prematurely fire with watermark (Long.MAX_VALUE - 1500)
+ Assert.assertTrue(window.maxTimestamp() + LATENESS < Long.MAX_VALUE - 1500);
+
+ // if we don't correctly prevent wrap-around in the garbage collection
+ // timers this watermark will clean our window state for the just-added
+ // element/window
testHarness.processWatermark(new Watermark(Long.MAX_VALUE - 1500));
- testHarness.processWatermark(new Watermark(endOfWindow));
- // guarantee that the cleanup would overflow
- Assert.assertTrue(endOfWindow + LATENESS < endOfWindow);
+ // this watermark is before the end timestamp of our only window
+ Assert.assertTrue(Long.MAX_VALUE - 1500 < window.maxTimestamp());
+ Assert.assertTrue(window.maxTimestamp() < Long.MAX_VALUE);
- // guarantee that the cleanup would have happened if it were not for the overflow check
- Assert.assertTrue(endOfWindow + LATENESS < Long.MAX_VALUE - 1500);
- Assert.assertTrue(Long.MAX_VALUE - 1500 < endOfWindow);
- Assert.assertTrue(endOfWindow < Long.MAX_VALUE);
+ // push in a watermark that will trigger computation of our window
+ testHarness.processWatermark(new Watermark(window.maxTimestamp()));
expected.add(new Watermark(Long.MAX_VALUE - 1500));
- expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), endOfWindow));
- expected.add(new Watermark(endOfWindow));
+ expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), window.maxTimestamp()));
+ expected.add(new Watermark(window.maxTimestamp()));
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, testHarness.getOutput(), new Tuple2ResultSortComparator());
testHarness.close();
@@ -1087,7 +1097,7 @@ public class WindowOperatorTest {
expected.add(new Watermark(initialTime + 1985));
- // a late that is added to an active window
+ // this will not be dropped because window.maxTimestamp() + allowedLateness > currentWatermark
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1980));
testHarness.processWatermark(new Watermark(initialTime + 1999));
@@ -1159,9 +1169,9 @@ public class WindowOperatorTest {
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 3001));
- // lateness is set to 0 and window_size = 3 sec and slide 1, the following 2 elements (2400) are late,
- // and assigned to windows ending at 2999, 3999, 4999.
- // The 2999 is dropped because it is already inactive (WM = 2999) but the rest are kept.
+ // lateness is set to 0 and window_size = 3 sec and slide 1, the following 2 elements (2400)
+ // are assigned to windows ending at 2999, 3999, 4999.
+ // The 2999 is dropped because it is already late (WM = 2999) but the rest are kept.
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2400));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2400));
@@ -1235,7 +1245,8 @@ public class WindowOperatorTest {
expected.add(new Watermark(4998));
- // late but added
+ // this will not be dropped because the session we're adding two has maxTimestamp
+ // after the current watermark
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
// new session
@@ -1244,25 +1255,21 @@ public class WindowOperatorTest {
expected.add(new Watermark(7400));
- // late element to merge the previous sessions
+ // this will merge the two sessions into one
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
testHarness.processWatermark(new Watermark(initialTime + 11501));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
testHarness.processWatermark(new Watermark(initialTime + 14600));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
- // NON-ACCUM TRIGGER
- // with 0 lateness this is dropped because it is assigned to a non-active window, (testDropDueToLatenessSessionZeroLateness)
- // with a small lateness (e.g. 10), this is assigned to window 10000-13000 (the 11600 is GCed after firing) (testDropDueToLatenessSessionWithLateness)
- // finally with a big lateness (e.g. 10000) the first window is not GCed so this merges all the windows so far (testDropDueToLatenessSessionWithHugeLateness)
-
+ // this is dropped as late
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
@@ -1283,6 +1290,11 @@ public class WindowOperatorTest {
@Test
public void testDropDueToLatenessSessionZeroLatenessAccum() throws Exception {
+ // same as testDropDueToLatenessSessionZeroLateness() but with an accumulating trigger, i.e.
+ // one that does not return FIRE_AND_PURGE when firing but just FIRE
+
+ // this has the same output as testDropDueToLatenessSessionZeroLateness() because
+ // accumulating/discarding does not make a difference with "allowed lateness" = 0.
final int GAP_SIZE = 3;
final long LATENESS = 0;
@@ -1326,7 +1338,8 @@ public class WindowOperatorTest {
expected.add(new Watermark(4998));
- // late but added
+ // this will not be dropped because the session we're adding two has maxTimestamp
+ // after the current watermark
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
// new session
@@ -1335,26 +1348,27 @@ public class WindowOperatorTest {
expected.add(new Watermark(7400));
- // late element to merge the previous sessions
+ // this will merge the two sessions into one
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
testHarness.processWatermark(new Watermark(initialTime + 11501));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
testHarness.processWatermark(new Watermark(initialTime + 14600));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
+ // this is dropped as late
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
testHarness.processWatermark(new Watermark(initialTime + 20000));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500l, 17500l), 17499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
expected.add(new Watermark(20000));
testHarness.processWatermark(new Watermark(initialTime + 100000));
@@ -1367,6 +1381,10 @@ public class WindowOperatorTest {
@Test
public void testDropDueToLatenessSessionWithLateness() throws Exception {
+
+ // this has the same output as testDropDueToLatenessSessionZeroLateness() because
+ // the allowed lateness is too small to make a difference
+
final int GAP_SIZE = 3;
final long LATENESS = 10;
@@ -1409,7 +1427,8 @@ public class WindowOperatorTest {
expected.add(new Watermark(4998));
- // late but added
+ // this will not be dropped because the session we're adding two has maxTimestamp
+ // after the current watermark
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
// new session
@@ -1418,26 +1437,27 @@ public class WindowOperatorTest {
expected.add(new Watermark(7400));
- // late element to merge the previous sessions
+ // this will merge the two sessions into one
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
testHarness.processWatermark(new Watermark(initialTime + 11501));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
testHarness.processWatermark(new Watermark(initialTime + 14600));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
+ // dropped as late
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
testHarness.processWatermark(new Watermark(initialTime + 20000));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500l, 17500l), 17499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
expected.add(new Watermark(20000));
testHarness.processWatermark(new Watermark(initialTime + 100000));
@@ -1450,6 +1470,10 @@ public class WindowOperatorTest {
@Test
public void testDropDueToLatenessSessionWithLatenessAccum() throws Exception {
+ // same as testDropDueToLatenessSessionWithLateness() but with an accumulating trigger, i.e.
+ // one that does not return FIRE_AND_PURGE when firing but just FIRE. The expected
+ // results are therefore slightly different.
+
final int GAP_SIZE = 3;
final long LATENESS = 10;
@@ -1492,7 +1516,8 @@ public class WindowOperatorTest {
expected.add(new Watermark(4998));
- // late but added
+ // this will not be dropped because the session we're adding two has maxTimestamp
+ // after the current watermark
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
// new session
@@ -1501,34 +1526,45 @@ public class WindowOperatorTest {
expected.add(new Watermark(7400));
- // late element to merge the previous sessions
+ // this will merge the two sessions into one
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
testHarness.processWatermark(new Watermark(initialTime + 11501));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
testHarness.processWatermark(new Watermark(initialTime + 14600));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
+ // because of the small allowed lateness and because the trigger is accumulating
+ // this will be merged into the session (11600-14600) and therefore will not
+ // be dropped as late
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
+
+ // adding ("key2", 1) extended the session to (10000-146000) for which
+ // maxTimestamp <= currentWatermark. Therefore, we immediately get a firing
+ // with the current version of EventTimeTrigger/EventTimeTriggerAccum
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-2", 10000L, 14600L), 14599));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
+
testHarness.processWatermark(new Watermark(initialTime + 20000));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-2", 10000l, 14600l), 14599));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-3", 10000l, 17500l), 17499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-3", 10000L, 17500L), 17499));
expected.add(new Watermark(20000));
testHarness.processWatermark(new Watermark(initialTime + 100000));
expected.add(new Watermark(100000));
- ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ actual = testHarness.getOutput();
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
testHarness.close();
}
@@ -1578,7 +1614,8 @@ public class WindowOperatorTest {
expected.add(new Watermark(4998));
- // late but added
+ // this will not be dropped because the session we're adding two has maxTimestamp
+ // after the current watermark
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
// new session
@@ -1587,34 +1624,38 @@ public class WindowOperatorTest {
expected.add(new Watermark(7400));
- // late element to merge the previous sessions
+ // this will merge the two sessions into one
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
testHarness.processWatermark(new Watermark(initialTime + 11501));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
testHarness.processWatermark(new Watermark(initialTime + 14600));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 10000L, 13000L), 12999));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
+
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
testHarness.processWatermark(new Watermark(initialTime + 20000));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 10000l, 13000l), 12999));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500l, 17500l), 17499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 14500L, 17500L), 17499));
expected.add(new Watermark(20000));
testHarness.processWatermark(new Watermark(initialTime + 100000));
expected.add(new Watermark(100000));
- ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ actual = testHarness.getOutput();
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
testHarness.close();
}
@@ -1663,7 +1704,8 @@ public class WindowOperatorTest {
expected.add(new Watermark(4998));
- // late but added
+ // this will not be dropped because the session we're adding two has maxTimestamp
+ // after the current watermark
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 4500));
// new session
@@ -1672,33 +1714,39 @@ public class WindowOperatorTest {
expected.add(new Watermark(7400));
- // late element to merge the previous sessions
+ // this will merge the two sessions into one
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 7000));
testHarness.processWatermark(new Watermark(initialTime + 11501));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000l, 11500l), 11499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-5", 1000L, 11500L), 11499));
expected.add(new Watermark(11501));
// new session
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 11600));
testHarness.processWatermark(new Watermark(initialTime + 14600));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600l, 14600l), 14599));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-1", 11600L, 14600L), 14599));
expected.add(new Watermark(14600));
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 10000));
+ // the maxTimestamp of the merged session is already late,
+ // so we get an immediate firing
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-7", 1000L, 14600L), 14599));
+
+ ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
+
testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 14500));
testHarness.processWatermark(new Watermark(initialTime + 20000));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-7", 1000l, 14600l), 14599));
- expected.add(new StreamRecord<>(new Tuple3<>("key2-8", 1000l, 17500l), 17499));
+ expected.add(new StreamRecord<>(new Tuple3<>("key2-8", 1000L, 17500L), 17499));
expected.add(new Watermark(20000));
testHarness.processWatermark(new Watermark(initialTime + 100000));
expected.add(new Watermark(100000));
- ConcurrentLinkedQueue<Object> actual = testHarness.getOutput();
+ actual = testHarness.getOutput();
TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple3ResultSortComparator());
testHarness.close();
}
@@ -1876,10 +1924,13 @@ public class WindowOperatorTest {
@Override
public TriggerResult onElement(Object element, long timestamp, TimeWindow window, TriggerContext ctx) throws Exception {
- ctx.registerEventTimeTimer(window.maxTimestamp());
- return (window.maxTimestamp() <= ctx.getCurrentWatermark()) ?
- TriggerResult.FIRE :
- TriggerResult.CONTINUE;
+ if (window.maxTimestamp() <= ctx.getCurrentWatermark()) {
+ // if the watermark is already past the window fire immediately
+ return TriggerResult.FIRE;
+ } else {
+ ctx.registerEventTimeTimer(window.maxTimestamp());
+ return TriggerResult.CONTINUE;
+ }
}
@Override
[3/6] flink git commit: [FLINK-3714] Remove Unneccesary Timer in
EventTimeTrigger
Posted by al...@apache.org.
[FLINK-3714] Remove Unneccesary Timer in EventTimeTrigger
In onElement() we registered a timer for the case where the watermark is
already past the end of the window and we're firing anyways. Now, only
add a timer if the watermark is not already past the end of the window.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0104a926
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0104a926
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0104a926
Branch: refs/heads/master
Commit: 0104a9260353e9a044504b65d480c83a7d3799fc
Parents: 34a8b03
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Jun 20 16:38:22 2016 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Tue Jun 21 12:16:19 2016 +0200
----------------------------------------------------------------------
.../api/windowing/triggers/EventTimeTrigger.java | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0104a926/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
index a87e436..96e862f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
@@ -35,11 +35,13 @@ public class EventTimeTrigger extends Trigger<Object, TimeWindow> {
@Override
public TriggerResult onElement(Object element, long timestamp, TimeWindow window, TriggerContext ctx) throws Exception {
- ctx.registerEventTimeTimer(window.maxTimestamp());
-
- return (window.maxTimestamp() <= ctx.getCurrentWatermark()) ?
- TriggerResult.FIRE_AND_PURGE :
- TriggerResult.CONTINUE;
+ if (window.maxTimestamp() <= ctx.getCurrentWatermark()) {
+ // if the watermark is already past the window fire immediately
+ return TriggerResult.FIRE_AND_PURGE;
+ } else {
+ ctx.registerEventTimeTimer(window.maxTimestamp());
+ return TriggerResult.CONTINUE;
+ }
}
@Override
[4/6] flink git commit: [FLINK-3714] Rename getCleanupTimeForWindow
to cleanupTime in WindowOperator
Posted by al...@apache.org.
[FLINK-3714] Rename getCleanupTimeForWindow to cleanupTime in WindowOperator
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/62c5a3c0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/62c5a3c0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/62c5a3c0
Branch: refs/heads/master
Commit: 62c5a3c07f8e10bad305316b2a1bf6d91620e9b4
Parents: 0104a92
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Jun 20 16:39:52 2016 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Tue Jun 21 12:16:26 2016 +0200
----------------------------------------------------------------------
.../runtime/operators/windowing/WindowOperator.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/62c5a3c0/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
index 95ad1b0..bad1a22 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
@@ -323,7 +323,7 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
}
});
- // check if the window is already inactive
+ // drop if the window is already late
if (isLate(actualWindow)) {
LOG.info("Dropped element " + element+ " for window " + actualWindow + " due to lateness.");
continue;
@@ -352,7 +352,7 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
} else {
for (W window: elementWindows) {
- // check if the window is already inactive
+ // drop if the window is already late
if (isLate(window)) {
LOG.info("Dropped element " + element + " for window " + window + " due to lateness.");
continue;
@@ -528,7 +528,7 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
* considered when triggering.
*/
protected boolean isLate(W window) {
- return (windowAssigner.isEventTime() && (getCleanupTimeForWindow(window) <= currentWatermark));
+ return (windowAssigner.isEventTime() && (cleanupTime(window) <= currentWatermark));
}
/**
@@ -537,7 +537,7 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
* the window whose state to discard
*/
protected void registerCleanupTimer(W window) {
- long cleanupTime = getCleanupTimeForWindow(window);
+ long cleanupTime = cleanupTime(window);
if (windowAssigner.isEventTime()) {
context.registerEventTimeTimer(cleanupTime);
} else {
@@ -551,7 +551,7 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
* the window whose state to discard
*/
protected void deleteCleanupTimer(W window) {
- long cleanupTime = getCleanupTimeForWindow(window);
+ long cleanupTime = cleanupTime(window);
if (windowAssigner.isEventTime()) {
context.deleteEventTimeTimer(cleanupTime);
} else {
@@ -568,7 +568,7 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
*
* @param window the window whose cleanup time we are computing.
*/
- private long getCleanupTimeForWindow(W window) {
+ private long cleanupTime(W window) {
long cleanupTime = window.maxTimestamp() + allowedLateness;
return cleanupTime >= window.maxTimestamp() ? cleanupTime : Long.MAX_VALUE;
}
@@ -585,7 +585,7 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
* @return {@code true} if it is time to clean up the window state, {@code false} otherwise.
*/
protected final boolean isCleanupTime(W window, long time) {
- long cleanupTime = getCleanupTimeForWindow(window);
+ long cleanupTime = cleanupTime(window);
return cleanupTime == time;
}
[2/6] flink git commit: [FLINK-3714] Add Support for "Allowed
Lateness"
Posted by al...@apache.org.
[FLINK-3714] Add Support for "Allowed Lateness"
Handle late elements and take care
of cleaning the window state.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/34a8b03d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/34a8b03d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/34a8b03d
Branch: refs/heads/master
Commit: 34a8b03d2ad40db7dc00fa47923b96374c289838
Parents: f2e9c52
Author: kl0u <kk...@gmail.com>
Authored: Tue May 31 17:13:58 2016 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Tue Jun 21 12:14:06 2016 +0200
----------------------------------------------------------------------
.../api/datastream/AllWindowedStream.java | 59 +-
.../api/datastream/WindowedStream.java | 53 +-
.../assigners/EventTimeSessionWindows.java | 5 +
.../api/windowing/assigners/GlobalWindows.java | 5 +
.../assigners/ProcessingTimeSessionWindows.java | 5 +
.../assigners/SlidingEventTimeWindows.java | 5 +
.../assigners/SlidingProcessingTimeWindows.java | 5 +
.../assigners/TumblingEventTimeWindows.java | 5 +
.../TumblingProcessingTimeWindows.java | 5 +
.../api/windowing/assigners/WindowAssigner.java | 6 +
.../api/windowing/evictors/CountEvictor.java | 2 +-
.../windowing/triggers/EventTimeTrigger.java | 9 +-
.../windowing/EvictingWindowOperator.java | 218 +++--
.../operators/windowing/WindowOperator.java | 327 ++++---
.../windowing/EvictingWindowOperatorTest.java | 75 +-
.../operators/windowing/WindowOperatorTest.java | 873 ++++++++++++++++++-
.../streaming/api/scala/AllWindowedStream.scala | 17 +-
.../streaming/api/scala/WindowedStream.scala | 15 +
18 files changed, 1480 insertions(+), 209 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
index 5a1b56d..e5dacc0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/AllWindowedStream.java
@@ -42,6 +42,7 @@ import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner;
import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;
import org.apache.flink.streaming.api.windowing.evictors.Evictor;
+import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.triggers.Trigger;
import org.apache.flink.streaming.api.windowing.windows.Window;
import org.apache.flink.streaming.runtime.operators.windowing.EvictingWindowOperator;
@@ -87,6 +88,8 @@ public class AllWindowedStream<T, W extends Window> {
/** The evictor that is used for evicting elements before window evaluation. */
private Evictor<? super T, ? super W> evictor;
+ /** The user-specified allowed lateness. */
+ private long allowedLateness = Long.MAX_VALUE;
@PublicEvolving
public AllWindowedStream(DataStream<T> input,
@@ -110,6 +113,26 @@ public class AllWindowedStream<T, W extends Window> {
}
/**
+ * Sets the allowed lateness to a user-specified value.
+ * If not explicitly set, the allowed lateness is {@link Long#MAX_VALUE}.
+ * Setting the allowed lateness is only valid for event-time windows.
+ * If a value different than 0 is provided with a processing-time
+ * {@link WindowAssigner}, then an exception is thrown.
+ */
+ @PublicEvolving
+ public AllWindowedStream<T, W> allowedLateness(Time lateness) {
+ long millis = lateness.toMilliseconds();
+ if (allowedLateness < 0) {
+ throw new IllegalArgumentException("The allowed lateness cannot be negative.");
+ } else if (allowedLateness != 0 && !windowAssigner.isEventTime()) {
+ throw new IllegalArgumentException("Setting the allowed lateness is only valid for event-time windows.");
+ } else {
+ this.allowedLateness = millis;
+ }
+ return this;
+ }
+
+ /**
* Sets the {@code Evictor} that should be used to evict elements from a window before emission.
*
* <p>
@@ -251,14 +274,16 @@ public class AllWindowedStream<T, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
- operator = new EvictingWindowOperator<>(windowAssigner,
+ operator =
+ new EvictingWindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalIterableAllWindowFunction<>(function),
trigger,
- evictor);
+ evictor,
+ allowedLateness);
} else {
ListStateDescriptor<T> stateDesc = new ListStateDescriptor<>("window-contents",
@@ -266,13 +291,15 @@ public class AllWindowedStream<T, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
- operator = new WindowOperator<>(windowAssigner,
+ operator =
+ new WindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalIterableAllWindowFunction<>(function),
- trigger);
+ trigger,
+ allowedLateness);
}
return input.transform(opName, resultType, operator).setParallelism(1);
@@ -335,14 +362,16 @@ public class AllWindowedStream<T, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
- operator = new EvictingWindowOperator<>(windowAssigner,
+ operator =
+ new EvictingWindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalIterableAllWindowFunction<>(new ReduceApplyAllWindowFunction<>(reduceFunction, function)),
trigger,
- evictor);
+ evictor,
+ allowedLateness);
} else {
ReducingStateDescriptor<T> stateDesc = new ReducingStateDescriptor<>("window-contents",
@@ -351,13 +380,15 @@ public class AllWindowedStream<T, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
- operator = new WindowOperator<>(windowAssigner,
+ operator =
+ new WindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalSingleValueAllWindowFunction<>(function),
- trigger);
+ trigger,
+ allowedLateness);
}
return input.transform(opName, resultType, operator).setParallelism(1);
@@ -425,14 +456,16 @@ public class AllWindowedStream<T, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
- operator = new EvictingWindowOperator<>(windowAssigner,
+ operator =
+ new EvictingWindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalIterableAllWindowFunction<>(new FoldApplyAllWindowFunction<>(initialValue, foldFunction, function)),
trigger,
- evictor);
+ evictor,
+ allowedLateness);
} else {
FoldingStateDescriptor<T, R> stateDesc = new FoldingStateDescriptor<>("window-contents",
@@ -442,13 +475,15 @@ public class AllWindowedStream<T, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
- operator = new WindowOperator<>(windowAssigner,
+ operator =
+ new WindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalSingleValueAllWindowFunction<>(function),
- trigger);
+ trigger,
+ allowedLateness);
}
return input.transform(opName, resultType, operator).setParallelism(1);
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
index 84290b2..6110480 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/datastream/WindowedStream.java
@@ -45,6 +45,7 @@ import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeW
import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows;
import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;
import org.apache.flink.streaming.api.windowing.evictors.Evictor;
+import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.triggers.ProcessingTimeTrigger;
import org.apache.flink.streaming.api.windowing.triggers.Trigger;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
@@ -98,6 +99,8 @@ public class WindowedStream<T, K, W extends Window> {
/** The evictor that is used for evicting elements before window evaluation. */
private Evictor<? super T, ? super W> evictor;
+ /** The user-specified allowed lateness. */
+ private long allowedLateness = Long.MAX_VALUE;
@PublicEvolving
public WindowedStream(KeyedStream<T, K> input,
@@ -121,6 +124,26 @@ public class WindowedStream<T, K, W extends Window> {
}
/**
+ * Sets the allowed lateness to a user-specified value.
+ * If not explicitly set, the allowed lateness is {@link Long#MAX_VALUE}.
+ * Setting the allowed lateness is only valid for event-time windows.
+ * If a value different than 0 is provided with a processing-time
+ * {@link WindowAssigner}, then an exception is thrown.
+ */
+ @PublicEvolving
+ public WindowedStream<T, K, W> allowedLateness(Time lateness) {
+ long millis = lateness.toMilliseconds();
+ if (allowedLateness < 0) {
+ throw new IllegalArgumentException("The allowed lateness cannot be negative.");
+ } else if (allowedLateness != 0 && !windowAssigner.isEventTime()) {
+ throw new IllegalArgumentException("Setting the allowed lateness is only valid for event-time windows.");
+ } else {
+ this.allowedLateness = millis;
+ }
+ return this;
+ }
+
+ /**
* Sets the {@code Evictor} that should be used to evict elements from a window before emission.
*
* <p>
@@ -272,14 +295,16 @@ public class WindowedStream<T, K, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
- operator = new EvictingWindowOperator<>(windowAssigner,
+ operator =
+ new EvictingWindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalIterableWindowFunction<>(function),
trigger,
- evictor);
+ evictor,
+ allowedLateness);
} else {
ListStateDescriptor<T> stateDesc = new ListStateDescriptor<>("window-contents",
@@ -287,13 +312,15 @@ public class WindowedStream<T, K, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
- operator = new WindowOperator<>(windowAssigner,
+ operator =
+ new WindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalIterableWindowFunction<>(function),
- trigger);
+ trigger,
+ allowedLateness);
}
return input.transform(opName, resultType, operator);
@@ -356,14 +383,16 @@ public class WindowedStream<T, K, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
- operator = new EvictingWindowOperator<>(windowAssigner,
+ operator =
+ new EvictingWindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalIterableWindowFunction<>(new ReduceApplyWindowFunction<>(reduceFunction, function)),
trigger,
- evictor);
+ evictor,
+ allowedLateness);
} else {
ReducingStateDescriptor<T> stateDesc = new ReducingStateDescriptor<>("window-contents",
@@ -372,13 +401,15 @@ public class WindowedStream<T, K, W extends Window> {
opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
- operator = new WindowOperator<>(windowAssigner,
+ operator =
+ new WindowOperator<>(windowAssigner,
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()),
keySel,
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(function),
- trigger);
+ trigger,
+ allowedLateness);
}
return input.transform(opName, resultType, operator);
@@ -453,7 +484,8 @@ public class WindowedStream<T, K, W extends Window> {
stateDesc,
new InternalIterableWindowFunction<>(new FoldApplyWindowFunction<>(initialValue, foldFunction, function)),
trigger,
- evictor);
+ evictor,
+ allowedLateness);
} else {
FoldingStateDescriptor<T, R> stateDesc = new FoldingStateDescriptor<>("window-contents",
@@ -469,7 +501,8 @@ public class WindowedStream<T, K, W extends Window> {
input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
stateDesc,
new InternalSingleValueWindowFunction<>(function),
- trigger);
+ trigger,
+ allowedLateness);
}
return input.transform(opName, resultType, operator);
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
index ed5add5..64c14cd 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/EventTimeSessionWindows.java
@@ -81,6 +81,11 @@ public class EventTimeSessionWindows extends MergingWindowAssigner<Object, TimeW
return new TimeWindow.Serializer();
}
+ @Override
+ public boolean isEventTime() {
+ return true;
+ }
+
/**
* Merge overlapping {@link TimeWindow}s.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
index dcf440c..71101f6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/GlobalWindows.java
@@ -102,4 +102,9 @@ public class GlobalWindows extends WindowAssigner<Object, GlobalWindow> {
public TypeSerializer<GlobalWindow> getWindowSerializer(ExecutionConfig executionConfig) {
return new GlobalWindow.Serializer();
}
+
+ @Override
+ public boolean isEventTime() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
index 608ebbc..0e1682d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/ProcessingTimeSessionWindows.java
@@ -81,6 +81,11 @@ public class ProcessingTimeSessionWindows extends MergingWindowAssigner<Object,
return new TimeWindow.Serializer();
}
+ @Override
+ public boolean isEventTime() {
+ return false;
+ }
+
/**
* Merge overlapping {@link TimeWindow}s.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
index 89b216e..83511df 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingEventTimeWindows.java
@@ -109,4 +109,9 @@ public class SlidingEventTimeWindows extends WindowAssigner<Object, TimeWindow>
public TypeSerializer<TimeWindow> getWindowSerializer(ExecutionConfig executionConfig) {
return new TimeWindow.Serializer();
}
+
+ @Override
+ public boolean isEventTime() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
index 4b91986..d2b0707 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/SlidingProcessingTimeWindows.java
@@ -101,4 +101,9 @@ public class SlidingProcessingTimeWindows extends WindowAssigner<Object, TimeWin
public TypeSerializer<TimeWindow> getWindowSerializer(ExecutionConfig executionConfig) {
return new TimeWindow.Serializer();
}
+
+ @Override
+ public boolean isEventTime() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
index 1f61281..70432a6 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingEventTimeWindows.java
@@ -95,4 +95,9 @@ public class TumblingEventTimeWindows extends WindowAssigner<Object, TimeWindow>
public TypeSerializer<TimeWindow> getWindowSerializer(ExecutionConfig executionConfig) {
return new TimeWindow.Serializer();
}
+
+ @Override
+ public boolean isEventTime() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
index 436a9ed..3ec55d0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/TumblingProcessingTimeWindows.java
@@ -86,4 +86,9 @@ public class TumblingProcessingTimeWindows extends WindowAssigner<Object, TimeWi
public TypeSerializer<TimeWindow> getWindowSerializer(ExecutionConfig executionConfig) {
return new TimeWindow.Serializer();
}
+
+ @Override
+ public boolean isEventTime() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
index 683ed42..0b49bce 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/assigners/WindowAssigner.java
@@ -63,4 +63,10 @@ public abstract class WindowAssigner<T, W extends Window> implements Serializabl
* this {@code WindowAssigner}.
*/
public abstract TypeSerializer<W> getWindowSerializer(ExecutionConfig executionConfig);
+
+ /**
+ * Returns {@code true} if elements are assigned to windows based on event time,
+ * {@code false} otherwise.
+ */
+ public abstract boolean isEventTime();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
index e989cbc..dc82521 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/evictors/CountEvictor.java
@@ -22,7 +22,7 @@ import org.apache.flink.streaming.api.windowing.windows.Window;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
/**
- * An {@link Evictor} that keeps only a certain amount of elements.
+ * An {@link Evictor} that keeps up to a certain amount of elements.
*
* @param <W> The type of {@link Window Windows} on which this {@code Evictor} can operate.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
index 75c6a9d..a87e436 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/windowing/triggers/EventTimeTrigger.java
@@ -36,12 +36,17 @@ public class EventTimeTrigger extends Trigger<Object, TimeWindow> {
@Override
public TriggerResult onElement(Object element, long timestamp, TimeWindow window, TriggerContext ctx) throws Exception {
ctx.registerEventTimeTimer(window.maxTimestamp());
- return TriggerResult.CONTINUE;
+
+ return (window.maxTimestamp() <= ctx.getCurrentWatermark()) ?
+ TriggerResult.FIRE_AND_PURGE :
+ TriggerResult.CONTINUE;
}
@Override
public TriggerResult onEventTime(long time, TimeWindow window, TriggerContext ctx) {
- return TriggerResult.FIRE_AND_PURGE;
+ return time == window.maxTimestamp() ?
+ TriggerResult.FIRE_AND_PURGE :
+ TriggerResult.CONTINUE;
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
index 84ee0b9..fa1c894 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperator.java
@@ -29,6 +29,7 @@ import org.apache.flink.api.common.state.StateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.api.windowing.assigners.MergingWindowAssigner;
import org.apache.flink.streaming.api.windowing.assigners.WindowAssigner;
import org.apache.flink.streaming.api.windowing.evictors.Evictor;
@@ -70,8 +71,11 @@ public class EvictingWindowOperator<K, IN, OUT, W extends Window> extends Window
StateDescriptor<? extends ListState<StreamRecord<IN>>, ?> windowStateDescriptor,
InternalWindowFunction<Iterable<IN>, OUT, K, W> windowFunction,
Trigger<? super IN, ? super W> trigger,
- Evictor<? super IN, ? super W> evictor) {
- super(windowAssigner, windowSerializer, keySelector, keySerializer, null, windowFunction, trigger);
+ Evictor<? super IN, ? super W> evictor,
+ long allowedLateness) {
+
+ super(windowAssigner, windowSerializer, keySelector,
+ keySerializer, null, windowFunction, trigger, allowedLateness);
this.evictor = requireNonNull(evictor);
this.windowStateDescriptor = windowStateDescriptor;
}
@@ -79,8 +83,10 @@ public class EvictingWindowOperator<K, IN, OUT, W extends Window> extends Window
@Override
@SuppressWarnings("unchecked")
public void processElement(StreamRecord<IN> element) throws Exception {
- Collection<W> elementWindows = windowAssigner.assignWindows(element.getValue(),
- element.getTimestamp());
+
+ Collection<W> elementWindows = windowAssigner.assignWindows(
+ element.getValue(),
+ element.getTimestamp());
final K key = (K) getStateBackend().getCurrentKey();
@@ -112,20 +118,27 @@ public class EvictingWindowOperator<K, IN, OUT, W extends Window> extends Window
for (W m : mergedWindows) {
context.window = m;
context.clear();
+ deleteCleanupTimer(m);
}
// merge the merged state windows into the newly resulting state window
- getStateBackend().mergePartitionedStates(stateWindowResult,
- mergedStateWindows,
- windowSerializer,
- (StateDescriptor<? extends MergingState<?, ?>, ?>) windowStateDescriptor);
+ getStateBackend().mergePartitionedStates(
+ stateWindowResult,
+ mergedStateWindows,
+ windowSerializer,
+ (StateDescriptor<? extends MergingState<?, ?>, ?>) windowStateDescriptor);
}
});
+ // check if the window is already inactive
+ if (isLate(actualWindow)) {
+ LOG.info("Dropped element " + element + " for window " + actualWindow + " due to lateness.");
+ continue;
+ }
+
W stateWindow = mergingWindows.getStateWindow(actualWindow);
- ListState<StreamRecord<IN>> windowState = getPartitionedState(stateWindow,
- windowSerializer,
- windowStateDescriptor);
+ ListState<StreamRecord<IN>> windowState = getPartitionedState(
+ stateWindow, windowSerializer, windowStateDescriptor);
windowState.add(element);
context.key = key;
@@ -134,77 +147,166 @@ public class EvictingWindowOperator<K, IN, OUT, W extends Window> extends Window
// we might have already fired because of a merge but still call onElement
// on the (possibly merged) window
TriggerResult triggerResult = context.onElement(element);
-
- TriggerResult combinedTriggerResult = TriggerResult.merge(triggerResult,
- mergeTriggerResult.f0);
-
- processTriggerResult(combinedTriggerResult, actualWindow);
+ TriggerResult combinedTriggerResult = TriggerResult.merge(triggerResult, mergeTriggerResult.f0);
+ fireOrContinue(combinedTriggerResult, actualWindow, windowState);
+
+ if (combinedTriggerResult.isPurge()) {
+ cleanup(actualWindow, windowState, mergingWindows);
+ } else {
+ registerCleanupTimer(actualWindow);
+ }
}
} else {
for (W window : elementWindows) {
- ListState<StreamRecord<IN>> windowState = getPartitionedState(window,
- windowSerializer,
- windowStateDescriptor);
+ // check if the window is already inactive
+ if (isLate(window)) {
+ LOG.info("Dropped element " + element + " for window " + window + " due to lateness.");
+ continue;
+ }
+ ListState<StreamRecord<IN>> windowState = getPartitionedState(
+ window, windowSerializer, windowStateDescriptor);
windowState.add(element);
context.key = key;
context.window = window;
+
TriggerResult triggerResult = context.onElement(element);
+ fireOrContinue(triggerResult, window, windowState);
- processTriggerResult(triggerResult, window);
+ if (triggerResult.isPurge()) {
+ cleanup(window, windowState, null);
+ } else {
+ registerCleanupTimer(window);
+ }
}
}
}
@Override
- @SuppressWarnings("unchecked,rawtypes")
- protected void processTriggerResult(TriggerResult triggerResult, W window) throws Exception {
- if (!triggerResult.isFire() && !triggerResult.isPurge()) {
- // do nothing
- return;
- }
+ public void processWatermark(Watermark mark) throws Exception {
+ boolean fire;
+ do {
+ Timer<K, W> timer = watermarkTimersQueue.peek();
+ if (timer != null && timer.timestamp <= mark.getTimestamp()) {
+ fire = true;
+
+ watermarkTimers.remove(timer);
+ watermarkTimersQueue.remove();
+
+ context.key = timer.key;
+ context.window = timer.window;
+ setKeyContext(timer.key);
+
+ ListState<StreamRecord<IN>> windowState;
+ MergingWindowSet<W> mergingWindows = null;
+
+ if (windowAssigner instanceof MergingWindowAssigner) {
+ mergingWindows = getMergingWindowSet();
+ W stateWindow = mergingWindows.getStateWindow(context.window);
+ windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
+ } else {
+ windowState = getPartitionedState(context.window, windowSerializer, windowStateDescriptor);
+ }
+
+ TriggerResult triggerResult = context.onEventTime(timer.timestamp);
+ fireOrContinue(triggerResult, context.window, windowState);
+
+ if (triggerResult.isPurge() || (windowAssigner.isEventTime() && isCleanupTime(timer.window, timer.timestamp))) {
+ cleanup(timer.window, windowState, mergingWindows);
+ }
+
+ } else {
+ fire = false;
+ }
+ } while (fire);
- ListState<StreamRecord<IN>> windowState;
+ output.emitWatermark(mark);
- MergingWindowSet<W> mergingWindows = null;
+ this.currentWatermark = mark.getTimestamp();
+ }
- if (windowAssigner instanceof MergingWindowAssigner) {
- mergingWindows = getMergingWindowSet();
- W stateWindow = mergingWindows.getStateWindow(window);
- windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
+ @Override
+ public void trigger(long time) throws Exception {
+ boolean fire;
- } else {
- windowState = getPartitionedState(window, windowSerializer, windowStateDescriptor);
- }
+ //Remove information about the triggering task
+ processingTimeTimerFutures.remove(time);
+ processingTimeTimerTimestamps.remove(time, processingTimeTimerTimestamps.count(time));
- if (triggerResult.isFire()) {
- timestampedCollector.setAbsoluteTimestamp(window.maxTimestamp());
- Iterable<StreamRecord<IN>> contents = windowState.get();
-
- // Work around type system restrictions...
- int toEvict = evictor.evict((Iterable) contents, Iterables.size(contents), context.window);
-
- FluentIterable<IN> projectedContents = FluentIterable
- .from(contents)
- .skip(toEvict)
- .transform(new Function<StreamRecord<IN>, IN>() {
- @Override
- public IN apply(StreamRecord<IN> input) {
- return input.getValue();
- }
- });
- userFunction.apply(context.key, context.window, projectedContents, timestampedCollector);
- }
- if (triggerResult.isPurge()) {
- windowState.clear();
- if (mergingWindows != null) {
- mergingWindows.retireWindow(window);
+ do {
+ Timer<K, W> timer = processingTimeTimersQueue.peek();
+ if (timer != null && timer.timestamp <= time) {
+ fire = true;
+
+ processingTimeTimers.remove(timer);
+ processingTimeTimersQueue.remove();
+
+ context.key = timer.key;
+ context.window = timer.window;
+ setKeyContext(timer.key);
+
+ ListState<StreamRecord<IN>> windowState;
+ MergingWindowSet<W> mergingWindows = null;
+
+ if (windowAssigner instanceof MergingWindowAssigner) {
+ mergingWindows = getMergingWindowSet();
+ W stateWindow = mergingWindows.getStateWindow(context.window);
+ windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
+ } else {
+ windowState = getPartitionedState(context.window, windowSerializer, windowStateDescriptor);
+ }
+
+ TriggerResult triggerResult = context.onProcessingTime(timer.timestamp);
+ fireOrContinue(triggerResult, context.window, windowState);
+
+ if (triggerResult.isPurge() || (!windowAssigner.isEventTime() && isCleanupTime(timer.window, timer.timestamp))) {
+ cleanup(timer.window, windowState, mergingWindows);
+ }
+
+ } else {
+ fire = false;
}
- context.clear();
+ } while (fire);
+ }
+
+ private void fireOrContinue(TriggerResult triggerResult,
+ W window,
+ ListState<StreamRecord<IN>> windowState) throws Exception {
+ if (!triggerResult.isFire()) {
+ return;
+ }
+
+ timestampedCollector.setAbsoluteTimestamp(window.maxTimestamp());
+ Iterable<StreamRecord<IN>> contents = windowState.get();
+
+ // Work around type system restrictions...
+ int toEvict = evictor.evict((Iterable) contents, Iterables.size(contents), context.window);
+
+ FluentIterable<IN> projectedContents = FluentIterable
+ .from(contents)
+ .skip(toEvict)
+ .transform(new Function<StreamRecord<IN>, IN>() {
+ @Override
+ public IN apply(StreamRecord<IN> input) {
+ return input.getValue();
+ }
+ });
+ userFunction.apply(context.key, context.window, projectedContents, timestampedCollector);
+ }
+
+ private void cleanup(W window,
+ ListState<StreamRecord<IN>> windowState,
+ MergingWindowSet<W> mergingWindows) throws Exception {
+
+ windowState.clear();
+ if (mergingWindows != null) {
+ mergingWindows.retireWindow(window);
}
+ context.clear();
+ deleteCleanupTimer(window);
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
index 18020b3..95ad1b0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/operators/windowing/WindowOperator.java
@@ -59,6 +59,7 @@ import org.apache.flink.streaming.runtime.operators.Triggerable;
import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalWindowFunction;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.StreamTaskState;
+import org.apache.flink.util.Preconditions;
import java.io.IOException;
import java.io.ObjectInputStream;
@@ -78,9 +79,9 @@ import static java.util.Objects.requireNonNull;
* {@link Trigger}.
*
* <p>
- * When an element arrives it gets assigned a key using a {@link KeySelector} and it get's
+ * When an element arrives it gets assigned a key using a {@link KeySelector} and it gets
* assigned to zero or more windows using a {@link WindowAssigner}. Based on this the element
- * is put into panes. A pane is the bucket of elements that have the same key and same
+ * is put into panes. A pane is the bucket of elements that have the same key and belong to the same
* {@code Window}. An element can be in multiple panes of it was assigned to multiple windows by the
* {@code WindowAssigner}.
*
@@ -130,6 +131,16 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
*/
protected final TypeSerializer<W> windowSerializer;
+ /**
+ * The allowed lateness for elements. This is used for:
+ * <ul>
+ * <li>Deciding if an element should be dropped from a window due to lateness.
+ * <li>Clearing the state of a window if the system time passes the
+ * {@code window.maxTimestamp + allowedLateness} landmark.
+ * </ul>
+ */
+ protected final long allowedLateness;
+
// ------------------------------------------------------------------------
// State that is not checkpointed
// ------------------------------------------------------------------------
@@ -177,7 +188,8 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
TypeSerializer<K> keySerializer,
StateDescriptor<? extends AppendingState<IN, ACC>, ?> windowStateDescriptor,
InternalWindowFunction<ACC, OUT, K, W> windowFunction,
- Trigger<? super IN, ? super W> trigger) {
+ Trigger<? super IN, ? super W> trigger,
+ long allowedLateness) {
super(windowFunction);
@@ -189,6 +201,9 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
this.windowStateDescriptor = windowStateDescriptor;
this.trigger = requireNonNull(trigger);
+ Preconditions.checkArgument(allowedLateness >= 0);
+ this.allowedLateness = allowedLateness;
+
setChainingStrategy(ChainingStrategy.ALWAYS);
}
@@ -264,7 +279,10 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
@Override
@SuppressWarnings("unchecked")
public void processElement(StreamRecord<IN> element) throws Exception {
- Collection<W> elementWindows = windowAssigner.assignWindows(element.getValue(), element.getTimestamp());
+
+ Collection<W> elementWindows = windowAssigner.assignWindows(
+ element.getValue(),
+ element.getTimestamp());
final K key = (K) getStateBackend().getCurrentKey();
@@ -276,7 +294,6 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
// element because we always eagerly merge
final Tuple1<TriggerResult> mergeTriggerResult = new Tuple1<>(TriggerResult.CONTINUE);
-
// adding the new window might result in a merge, in that case the actualWindow
// is the merged window and we work with that. If we don't merge then
// actualWindow == window
@@ -294,18 +311,27 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
for (W m: mergedWindows) {
context.window = m;
context.clear();
+ deleteCleanupTimer(m);
}
// merge the merged state windows into the newly resulting state window
- getStateBackend().mergePartitionedStates(stateWindowResult,
- mergedStateWindows,
- windowSerializer,
- (StateDescriptor<? extends MergingState<?,?>, ?>) windowStateDescriptor);
+ getStateBackend().mergePartitionedStates(
+ stateWindowResult,
+ mergedStateWindows,
+ windowSerializer,
+ (StateDescriptor<? extends MergingState<?,?>, ?>) windowStateDescriptor);
}
});
+ // check if the window is already inactive
+ if (isLate(actualWindow)) {
+ LOG.info("Dropped element " + element+ " for window " + actualWindow + " due to lateness.");
+ continue;
+ }
+
W stateWindow = mergingWindows.getStateWindow(actualWindow);
- AppendingState<IN, ACC> windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
+ AppendingState<IN, ACC> windowState = getPartitionedState(
+ stateWindow, windowSerializer, windowStateDescriptor);
windowState.add(element.getValue());
context.key = key;
@@ -314,104 +340,46 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
// we might have already fired because of a merge but still call onElement
// on the (possibly merged) window
TriggerResult triggerResult = context.onElement(element);
-
TriggerResult combinedTriggerResult = TriggerResult.merge(triggerResult, mergeTriggerResult.f0);
+ fireOrContinue(combinedTriggerResult, actualWindow, windowState);
- processTriggerResult(combinedTriggerResult, actualWindow);
+ if (combinedTriggerResult.isPurge()) {
+ cleanup(actualWindow, windowState, mergingWindows);
+ } else {
+ registerCleanupTimer(actualWindow);
+ }
}
-
} else {
for (W window: elementWindows) {
- AppendingState<IN, ACC> windowState = getPartitionedState(window, windowSerializer,
- windowStateDescriptor);
+ // check if the window is already inactive
+ if (isLate(window)) {
+ LOG.info("Dropped element " + element + " for window " + window + " due to lateness.");
+ continue;
+ }
+ AppendingState<IN, ACC> windowState = getPartitionedState(
+ window, windowSerializer, windowStateDescriptor);
windowState.add(element.getValue());
context.key = key;
context.window = window;
- TriggerResult triggerResult = context.onElement(element);
-
- processTriggerResult(triggerResult, window);
- }
- }
- }
-
- /**
- * Retrieves the {@link MergingWindowSet} for the currently active key. The caller must
- * ensure that the correct key is set in the state backend.
- */
- @SuppressWarnings("unchecked")
- protected MergingWindowSet<W> getMergingWindowSet() throws Exception {
- MergingWindowSet<W> mergingWindows = mergingWindowsByKey.get((K) getStateBackend().getCurrentKey());
- if (mergingWindows == null) {
- // try to retrieve from state
-
- TupleSerializer<Tuple2<W, W>> tupleSerializer = new TupleSerializer<>((Class) Tuple2.class, new TypeSerializer[] {windowSerializer, windowSerializer} );
- ListStateDescriptor<Tuple2<W, W>> mergeStateDescriptor = new ListStateDescriptor<>("merging-window-set", tupleSerializer);
- ListState<Tuple2<W, W>> mergeState = getStateBackend().getPartitionedState(null, VoidSerializer.INSTANCE, mergeStateDescriptor);
-
- mergingWindows = new MergingWindowSet<>((MergingWindowAssigner<? super IN, W>) windowAssigner, mergeState);
- mergeState.clear();
-
- mergingWindowsByKey.put((K) getStateBackend().getCurrentKey(), mergingWindows);
- }
- return mergingWindows;
- }
-
-
- /**
- * Process {@link TriggerResult} for the currently active key and the given window. The caller
- * must ensure that the correct key is set in the state backend and the context object.
- */
- @SuppressWarnings("unchecked")
- protected void processTriggerResult(TriggerResult triggerResult, W window) throws Exception {
- if (!triggerResult.isFire() && !triggerResult.isPurge()) {
- // do nothing
- return;
- }
-
- AppendingState<IN, ACC> windowState;
-
- MergingWindowSet<W> mergingWindows = null;
-
- if (windowAssigner instanceof MergingWindowAssigner) {
- mergingWindows = getMergingWindowSet();
- W stateWindow = mergingWindows.getStateWindow(window);
- windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
-
- } else {
- windowState = getPartitionedState(window, windowSerializer, windowStateDescriptor);
- }
-
- if (triggerResult.isFire()) {
- timestampedCollector.setAbsoluteTimestamp(window.maxTimestamp());
- ACC contents = windowState.get();
- userFunction.apply(context.key, context.window, contents, timestampedCollector);
+ TriggerResult triggerResult = context.onElement(element);
+ fireOrContinue(triggerResult, window, windowState);
- }
- if (triggerResult.isPurge()) {
- windowState.clear();
- if (mergingWindows != null) {
- mergingWindows.retireWindow(window);
+ if (triggerResult.isPurge()) {
+ cleanup(window, windowState, null);
+ } else {
+ registerCleanupTimer(window);
+ }
}
- context.clear();
}
}
@Override
- public final void processWatermark(Watermark mark) throws Exception {
- processTriggersFor(mark);
-
- output.emitWatermark(mark);
-
- this.currentWatermark = mark.getTimestamp();
- }
-
- private void processTriggersFor(Watermark mark) throws Exception {
+ public void processWatermark(Watermark mark) throws Exception {
boolean fire;
-
do {
Timer<K, W> timer = watermarkTimersQueue.peek();
if (timer != null && timer.timestamp <= mark.getTimestamp()) {
@@ -423,16 +391,37 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
context.key = timer.key;
context.window = timer.window;
setKeyContext(timer.key);
+
+ AppendingState<IN, ACC> windowState;
+ MergingWindowSet<W> mergingWindows = null;
+
+ if (windowAssigner instanceof MergingWindowAssigner) {
+ mergingWindows = getMergingWindowSet();
+ W stateWindow = mergingWindows.getStateWindow(context.window);
+ windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
+ } else {
+ windowState = getPartitionedState(context.window, windowSerializer, windowStateDescriptor);
+ }
+
TriggerResult triggerResult = context.onEventTime(timer.timestamp);
- processTriggerResult(triggerResult, context.window);
+ fireOrContinue(triggerResult, context.window, windowState);
+
+ if (triggerResult.isPurge() || (windowAssigner.isEventTime() && isCleanupTime(timer.window, timer.timestamp))) {
+ cleanup(timer.window, windowState, mergingWindows);
+ }
+
} else {
fire = false;
}
} while (fire);
+
+ output.emitWatermark(mark);
+
+ this.currentWatermark = mark.getTimestamp();
}
@Override
- public final void trigger(long time) throws Exception {
+ public void trigger(long time) throws Exception {
boolean fire;
//Remove information about the triggering task
@@ -450,17 +439,154 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
context.key = timer.key;
context.window = timer.window;
setKeyContext(timer.key);
+
+ AppendingState<IN, ACC> windowState;
+ MergingWindowSet<W> mergingWindows = null;
+
+ if (windowAssigner instanceof MergingWindowAssigner) {
+ mergingWindows = getMergingWindowSet();
+ W stateWindow = mergingWindows.getStateWindow(context.window);
+ windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
+ } else {
+ windowState = getPartitionedState(context.window, windowSerializer, windowStateDescriptor);
+ }
+
TriggerResult triggerResult = context.onProcessingTime(timer.timestamp);
- processTriggerResult(triggerResult, context.window);
+ fireOrContinue(triggerResult, context.window, windowState);
+
+ if (triggerResult.isPurge() || (!windowAssigner.isEventTime() && isCleanupTime(timer.window, timer.timestamp))) {
+ cleanup(timer.window, windowState, mergingWindows);
+ }
+
} else {
fire = false;
}
} while (fire);
+ }
+
+ /**
+ * Cleans up the window state if the provided {@link TriggerResult} requires so, or if it
+ * is time to do so (see {@link #isCleanupTime(Window, long)}). The caller must ensure that the
+ * correct key is set in the state backend and the context object.
+ */
+ private void cleanup(W window,
+ AppendingState<IN, ACC> windowState,
+ MergingWindowSet<W> mergingWindows) throws Exception {
+ windowState.clear();
+ if (mergingWindows != null) {
+ mergingWindows.retireWindow(window);
+ }
+ context.clear();
+ deleteCleanupTimer(window);
+ }
+
+ /**
+ * Triggers the window computation if the provided {@link TriggerResult} requires so.
+ * The caller must ensure that the correct key is set in the state backend and the context object.
+ */
+ @SuppressWarnings("unchecked")
+ private void fireOrContinue(TriggerResult triggerResult,
+ W window,
+ AppendingState<IN, ACC> windowState) throws Exception {
+ if (!triggerResult.isFire()) {
+ return;
+ }
+
+ timestampedCollector.setAbsoluteTimestamp(window.maxTimestamp());
+ ACC contents = windowState.get();
+ userFunction.apply(context.key, context.window, contents, timestampedCollector);
+ }
+
+ /**
+ * Retrieves the {@link MergingWindowSet} for the currently active key.
+ * The caller must ensure that the correct key is set in the state backend.
+ */
+ @SuppressWarnings("unchecked")
+ protected MergingWindowSet<W> getMergingWindowSet() throws Exception {
+ MergingWindowSet<W> mergingWindows = mergingWindowsByKey.get((K) getStateBackend().getCurrentKey());
+ if (mergingWindows == null) {
+ // try to retrieve from state
+
+ TupleSerializer<Tuple2<W, W>> tupleSerializer = new TupleSerializer<>((Class) Tuple2.class, new TypeSerializer[] {windowSerializer, windowSerializer} );
+ ListStateDescriptor<Tuple2<W, W>> mergeStateDescriptor = new ListStateDescriptor<>("merging-window-set", tupleSerializer);
+ ListState<Tuple2<W, W>> mergeState = getStateBackend().getPartitionedState(null, VoidSerializer.INSTANCE, mergeStateDescriptor);
+
+ mergingWindows = new MergingWindowSet<>((MergingWindowAssigner<? super IN, W>) windowAssigner, mergeState);
+ mergeState.clear();
+
+ mergingWindowsByKey.put((K) getStateBackend().getCurrentKey(), mergingWindows);
+ }
+ return mergingWindows;
+ }
- // Also check any watermark timers. We might have some in here since
- // Context.registerEventTimeTimer sets a trigger if an event-time trigger is registered
- // that is already behind the watermark.
- processTriggersFor(new Watermark(currentWatermark));
+ /**
+ * Decides if a window is currently late or not, based on the current
+ * watermark, i.e. the current event time, and the allowed lateness.
+ * @param window
+ * The collection of windows returned by the {@link WindowAssigner}.
+ * @return The windows (among the {@code eligibleWindows}) for which the element should still be
+ * considered when triggering.
+ */
+ protected boolean isLate(W window) {
+ return (windowAssigner.isEventTime() && (getCleanupTimeForWindow(window) <= currentWatermark));
+ }
+
+ /**
+ * Registers a timer to cleanup the content of the window.
+ * @param window
+ * the window whose state to discard
+ */
+ protected void registerCleanupTimer(W window) {
+ long cleanupTime = getCleanupTimeForWindow(window);
+ if (windowAssigner.isEventTime()) {
+ context.registerEventTimeTimer(cleanupTime);
+ } else {
+ context.registerProcessingTimeTimer(cleanupTime);
+ }
+ }
+
+ /**
+ * Deletes the cleanup timer set for the contents of the provided window.
+ * @param window
+ * the window whose state to discard
+ */
+ protected void deleteCleanupTimer(W window) {
+ long cleanupTime = getCleanupTimeForWindow(window);
+ if (windowAssigner.isEventTime()) {
+ context.deleteEventTimeTimer(cleanupTime);
+ } else {
+ context.deleteProcessingTimeTimer(cleanupTime);
+ }
+ }
+
+ /**
+ * Returns the cleanup time for a window, which is
+ * {@code window.maxTimestamp + allowedLateness}. In
+ * case this leads to a value greated than {@link Long#MAX_VALUE}
+ * then a cleanup time of {@link Long#MAX_VALUE} is
+ * returned.
+ *
+ * @param window the window whose cleanup time we are computing.
+ */
+ private long getCleanupTimeForWindow(W window) {
+ long cleanupTime = window.maxTimestamp() + allowedLateness;
+ return cleanupTime >= window.maxTimestamp() ? cleanupTime : Long.MAX_VALUE;
+ }
+
+ /**
+ * Decides if it is time to clean up the window state.
+ * Clean up time for a window is:
+ * <li> if it is event time, after the watermark passes the end of the window plus the user-specified allowed lateness
+ * <li> if it is processing time, after the processing time at the node passes the end of the window.
+ * @param window
+ * the window to clean
+ * @param time
+ * the current time (event or processing depending on the {@link WindowAssigner}
+ * @return {@code true} if it is time to clean up the window state, {@code false} otherwise.
+ */
+ protected final boolean isCleanupTime(W window, long time) {
+ long cleanupTime = getCleanupTimeForWindow(window);
+ return cleanupTime == time;
}
/**
@@ -562,13 +688,6 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
if (watermarkTimers.add(timer)) {
watermarkTimersQueue.add(timer);
}
-
- if (time <= currentWatermark) {
- // immediately schedule a trigger, so that we don't wait for the next
- // watermark update to fire the watermark trigger
- getRuntimeContext().registerTimer(System.currentTimeMillis(), WindowOperator.this);
- //No need to put it in processingTimeTimerFutures as this timer is never removed
- }
}
@Override
@@ -594,7 +713,6 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
if (watermarkTimers.remove(timer)) {
watermarkTimersQueue.remove(timer);
}
-
}
public TriggerResult onElement(StreamRecord<IN> element) throws Exception {
@@ -627,7 +745,6 @@ public class WindowOperator<K, IN, ACC, OUT, W extends Window>
}
}
-
/**
* Internal class for keeping track of in-flight timers.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/34a8b03d/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java
index 6af7ac4..dc71440 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/EvictingWindowOperatorTest.java
@@ -30,9 +30,13 @@ import org.apache.flink.streaming.api.functions.windowing.ReduceIterableWindowFu
import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.api.windowing.assigners.GlobalWindows;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.evictors.CountEvictor;
+import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.triggers.CountTrigger;
+import org.apache.flink.streaming.api.windowing.triggers.EventTimeTrigger;
import org.apache.flink.streaming.api.windowing.windows.GlobalWindow;
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.streaming.api.windowing.windows.Window;
import org.apache.flink.streaming.runtime.operators.windowing.functions.InternalIterableWindowFunction;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -45,6 +49,7 @@ import org.junit.Test;
import java.util.Comparator;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class EvictingWindowOperatorTest {
@@ -73,7 +78,8 @@ public class EvictingWindowOperatorTest {
stateDesc,
new InternalIterableWindowFunction<>(new ReduceIterableWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>(new SumReducer())),
CountTrigger.of(WINDOW_SLIDE),
- CountEvictor.of(WINDOW_SIZE));
+ CountEvictor.of(WINDOW_SIZE),
+ 0);
operator.setInputType(inputType, new ExecutionConfig());
@@ -144,7 +150,8 @@ public class EvictingWindowOperatorTest {
stateDesc,
new InternalIterableWindowFunction<>(new RichSumReducer<GlobalWindow>(closeCalled)),
CountTrigger.of(WINDOW_SLIDE),
- CountEvictor.of(WINDOW_SIZE));
+ CountEvictor.of(WINDOW_SIZE),
+ 0);
operator.setInputType(inputType, new ExecutionConfig());
@@ -194,7 +201,69 @@ public class EvictingWindowOperatorTest {
Assert.assertEquals("Close was not called.", 1, closeCalled.get());
}
- // ------------------------------------------------------------------------
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTumblingWindowWithApply() throws Exception {
+ AtomicInteger closeCalled = new AtomicInteger(0);
+
+ final int WINDOW_SIZE = 4;
+
+ TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
+
+ ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc = new ListStateDescriptor<>("window-contents",
+ new StreamRecordSerializer<>(inputType.createSerializer(new ExecutionConfig())));
+
+ EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new EvictingWindowOperator<>(
+ TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
+ new TimeWindow.Serializer(),
+ new TupleKeySelector(),
+ BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
+ stateDesc,
+ new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>(closeCalled)),
+ EventTimeTrigger.create(),
+ CountEvictor.of(WINDOW_SIZE),
+ 0);
+
+ operator.setInputType(inputType, new ExecutionConfig());
+
+ OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness =
+ new OneInputStreamOperatorTestHarness<>(operator);
+
+ testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
+
+ long initialTime = 0L;
+
+ testHarness.open();
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 100));
+
+ testHarness.processWatermark(new Watermark(1999));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1997));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1998));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310)); // not late but more than 4
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310));
+
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));
+ testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));
+
+ testHarness.processWatermark(new Watermark(3999)); // now is the evictor
+
+ ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
+ expectedOutput.add(new Watermark(1999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), 3999));
+ expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 3999));
+ expectedOutput.add(new Watermark(3999));
+
+
+ TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(),
+ new EvictingWindowOperatorTest.ResultSortComparator());
+ testHarness.close();
+ }
+
+ // ------------------------------------------------------------------------
// UDFs
// ------------------------------------------------------------------------