You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sj...@apache.org on 2020/05/04 14:56:32 UTC
[flink] branch release-1.10 updated:
[FLINK-17499][state-processor-api] LazyTimerService used to register timers
via State Processing API incorectly mixes event time timers with processing
time timers
This is an automated email from the ASF dual-hosted git repository.
sjwiesman pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.10 by this push:
new ebba158 [FLINK-17499][state-processor-api] LazyTimerService used to register timers via State Processing API incorectly mixes event time timers with processing time timers
ebba158 is described below
commit ebba1589d8b407f53bcc4e325cd63eaa6b30870b
Author: Seth Wiesman <sj...@gmail.com>
AuthorDate: Sun May 3 13:24:25 2020 -0500
[FLINK-17499][state-processor-api] LazyTimerService used to register timers via State Processing API incorectly mixes event time timers with processing time timers
This closes #11980
---
.../api/output/operators/LazyTimerService.java | 4 +-
.../output/KeyedStateBootstrapOperatorTest.java | 110 +++++++++++++++++----
2 files changed, 93 insertions(+), 21 deletions(-)
diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/operators/LazyTimerService.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/operators/LazyTimerService.java
index a818b01..eb9b3b9 100644
--- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/operators/LazyTimerService.java
+++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/operators/LazyTimerService.java
@@ -59,13 +59,13 @@ public class LazyTimerService implements TimerService {
@Override
public void registerProcessingTimeTimer(long time) {
ensureInitialized();
- internalTimerService.registerEventTimeTimer(VoidNamespace.INSTANCE, time);
+ internalTimerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, time);
}
@Override
public void registerEventTimeTimer(long time) {
ensureInitialized();
- internalTimerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, time);
+ internalTimerService.registerEventTimeTimer(VoidNamespace.INSTANCE, time);
}
@Override
diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java
index 0a39930..62f011f 100644
--- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java
+++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java
@@ -22,15 +22,20 @@ import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
import org.apache.flink.state.api.functions.KeyedStateBootstrapFunction;
import org.apache.flink.state.api.output.operators.KeyedStateBootstrapOperator;
+import org.apache.flink.streaming.api.TimeDomain;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.streaming.api.operators.KeyedProcessOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.StreamMap;
import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
+import org.apache.flink.util.Collector;
import org.hamcrest.Matchers;
import org.junit.Assert;
@@ -43,52 +48,119 @@ import org.junit.rules.TemporaryFolder;
*/
public class KeyedStateBootstrapOperatorTest {
- private static final ValueStateDescriptor<Long> descriptor = new ValueStateDescriptor<Long>("state", Types.LONG);
+ private static final ValueStateDescriptor<Long> descriptor = new ValueStateDescriptor<>("state", Types.LONG);
+
+ private static final Long EVENT_TIMER = Long.MAX_VALUE - 1;
+
+ private static final Long PROC_TIMER = Long.MAX_VALUE - 2;
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@Test
- public void testNonTimerStatesRestorableByNonProcessesOperator() throws Exception {
+ public void testTimerStateRestorable() throws Exception {
Path path = new Path(folder.newFolder().toURI());
OperatorSubtaskState state;
- KeyedStateBootstrapOperator<Long, Long> bootstrapOperator = new KeyedStateBootstrapOperator<>(0L, path, new SimpleBootstrapFunction());
+ KeyedStateBootstrapOperator<Long, Long> bootstrapOperator = new KeyedStateBootstrapOperator<>(0L, path, new TimerBootstrapFunction());
try (KeyedOneInputStreamOperatorTestHarness<Long, Long, TaggedOperatorSubtaskState> harness = getHarness(bootstrapOperator)) {
- harness.open();
-
- harness.processElement(1L, 0L);
- harness.processElement(2L, 0L);
- harness.processElement(3L, 0L);
- bootstrapOperator.endInput();
-
- state = harness.extractOutputValues().get(0).state;
+ processElements(harness, 1L, 2L, 3L);
+ state = getState(bootstrapOperator, harness);
}
- StreamMap<Long, Long> mapOperator = new StreamMap<>(new StreamingFunction());
- try (KeyedOneInputStreamOperatorTestHarness<Long, Long, Long> harness = getHarness(mapOperator)) {
+ KeyedProcessOperator<Long, Long, Tuple3<Long, Long, TimeDomain>> procOperator = new KeyedProcessOperator<>(new SimpleProcessFunction());
+ try (KeyedOneInputStreamOperatorTestHarness<Long, Long, Tuple3<Long, Long, TimeDomain>> harness = getHarness(procOperator, state)) {
+ harness.processWatermark(EVENT_TIMER);
+ harness.setProcessingTime(PROC_TIMER);
+
+ assertHarnessOutput(harness,
+ Tuple3.of(1L, EVENT_TIMER, TimeDomain.EVENT_TIME),
+ Tuple3.of(2L, EVENT_TIMER, TimeDomain.EVENT_TIME),
+ Tuple3.of(3L, EVENT_TIMER, TimeDomain.EVENT_TIME),
+ Tuple3.of(1L, PROC_TIMER, TimeDomain.PROCESSING_TIME),
+ Tuple3.of(2L, PROC_TIMER, TimeDomain.PROCESSING_TIME),
+ Tuple3.of(3L, PROC_TIMER, TimeDomain.PROCESSING_TIME));
+ }
+ }
- harness.initializeState(state);
- harness.open();
+ @Test
+ public void testNonTimerStatesRestorableByNonProcessesOperator() throws Exception {
+ Path path = new Path(folder.newFolder().toURI());
- harness.processElement(1L, 0L);
- harness.processElement(2L, 0L);
- harness.processElement(3L, 0L);
+ OperatorSubtaskState state;
+ KeyedStateBootstrapOperator<Long, Long> bootstrapOperator = new KeyedStateBootstrapOperator<>(0L, path, new SimpleBootstrapFunction());
+ try (KeyedOneInputStreamOperatorTestHarness<Long, Long, TaggedOperatorSubtaskState> harness = getHarness(bootstrapOperator)) {
+ processElements(harness, 1L, 2L, 3L);
+ state = getState(bootstrapOperator, harness);
+ }
- Assert.assertThat(harness.extractOutputValues(), Matchers.containsInAnyOrder(1L, 2L, 3L));
+ StreamMap<Long, Long> mapOperator = new StreamMap<>(new StreamingFunction());
+ try (KeyedOneInputStreamOperatorTestHarness<Long, Long, Long> harness = getHarness(mapOperator, state)) {
+ processElements(harness, 1L, 2L, 3L);
+ assertHarnessOutput(harness, 1L, 2L, 3L);
harness.snapshot(0L, 0L);
}
}
private <T> KeyedOneInputStreamOperatorTestHarness<Long, Long, T> getHarness(OneInputStreamOperator<Long, T> bootstrapOperator) throws Exception {
+ return getHarness(bootstrapOperator, null);
+ }
+
+ private <T> KeyedOneInputStreamOperatorTestHarness<Long, Long, T> getHarness(OneInputStreamOperator<Long, T> bootstrapOperator, OperatorSubtaskState state) throws Exception {
KeyedOneInputStreamOperatorTestHarness<Long, Long, T> harness = new KeyedOneInputStreamOperatorTestHarness<>(
bootstrapOperator, id -> id, Types.LONG, 128, 1, 0);
harness.setStateBackend(new RocksDBStateBackend(folder.newFolder().toURI()));
+ if (state != null) {
+ harness.initializeState(state);
+ }
+ harness.open();
return harness;
}
+ private <T> void processElements(KeyedOneInputStreamOperatorTestHarness<Long, Long, T> harness, Long... records) throws Exception {
+ for (Long record : records) {
+ harness.processElement(record, 0);
+ }
+ }
+
+ private OperatorSubtaskState getState(
+ KeyedStateBootstrapOperator<Long, Long> bootstrapOperator,
+ KeyedOneInputStreamOperatorTestHarness<Long, Long, TaggedOperatorSubtaskState> harness) throws Exception {
+ OperatorSubtaskState state;
+ bootstrapOperator.endInput();
+ state = harness.extractOutputValues().get(0).state;
+ return state;
+ }
+
+ private <T> void assertHarnessOutput(KeyedOneInputStreamOperatorTestHarness<Long, Long, T> harness, T... output) {
+ Assert.assertThat("The output from the operator does not match the expected values",
+ harness.extractOutputValues(), Matchers.containsInAnyOrder(output));
+ }
+
+ private static class TimerBootstrapFunction extends KeyedStateBootstrapFunction<Long, Long> {
+
+ @Override
+ public void processElement(Long value, Context ctx) throws Exception {
+ ctx.timerService().registerEventTimeTimer(EVENT_TIMER);
+ ctx.timerService().registerProcessingTimeTimer(PROC_TIMER);
+ }
+ }
+
+ private static class SimpleProcessFunction extends KeyedProcessFunction<Long, Long, Tuple3<Long, Long, TimeDomain>> {
+
+ @Override
+ public void processElement(Long value, Context ctx, Collector<Tuple3<Long, Long, TimeDomain>> out) throws Exception {
+
+ }
+
+ @Override
+ public void onTimer(long timestamp, OnTimerContext ctx, Collector<Tuple3<Long, Long, TimeDomain>> out) throws Exception {
+ out.collect(Tuple3.of(ctx.getCurrentKey(), timestamp, ctx.timeDomain()));
+ }
+ }
+
private static class SimpleBootstrapFunction extends KeyedStateBootstrapFunction<Long, Long> {
private ValueState<Long> state;