You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sj...@apache.org on 2020/05/04 14:56:32 UTC

[flink] branch release-1.10 updated: [FLINK-17499][state-processor-api] LazyTimerService used to register timers via State Processing API incorectly mixes event time timers with processing time timers

This is an automated email from the ASF dual-hosted git repository.

sjwiesman pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.10 by this push:
     new ebba158  [FLINK-17499][state-processor-api] LazyTimerService used to register timers via State Processing API incorectly mixes event time timers with processing time timers
ebba158 is described below

commit ebba1589d8b407f53bcc4e325cd63eaa6b30870b
Author: Seth Wiesman <sj...@gmail.com>
AuthorDate: Sun May 3 13:24:25 2020 -0500

    [FLINK-17499][state-processor-api] LazyTimerService used to register timers via State Processing API incorectly mixes event time timers with processing time timers
    
    This closes #11980
---
 .../api/output/operators/LazyTimerService.java     |   4 +-
 .../output/KeyedStateBootstrapOperatorTest.java    | 110 +++++++++++++++++----
 2 files changed, 93 insertions(+), 21 deletions(-)

diff --git a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/operators/LazyTimerService.java b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/operators/LazyTimerService.java
index a818b01..eb9b3b9 100644
--- a/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/operators/LazyTimerService.java
+++ b/flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/operators/LazyTimerService.java
@@ -59,13 +59,13 @@ public class LazyTimerService implements TimerService {
 	@Override
 	public void registerProcessingTimeTimer(long time) {
 		ensureInitialized();
-		internalTimerService.registerEventTimeTimer(VoidNamespace.INSTANCE, time);
+		internalTimerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, time);
 	}
 
 	@Override
 	public void registerEventTimeTimer(long time) {
 		ensureInitialized();
-		internalTimerService.registerProcessingTimeTimer(VoidNamespace.INSTANCE, time);
+		internalTimerService.registerEventTimeTimer(VoidNamespace.INSTANCE, time);
 	}
 
 	@Override
diff --git a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java
index 0a39930..62f011f 100644
--- a/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java
+++ b/flink-libraries/flink-state-processing-api/src/test/java/org/apache/flink/state/api/output/KeyedStateBootstrapOperatorTest.java
@@ -22,15 +22,20 @@ import org.apache.flink.api.common.functions.RichMapFunction;
 import org.apache.flink.api.common.state.ValueState;
 import org.apache.flink.api.common.state.ValueStateDescriptor;
 import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.contrib.streaming.state.RocksDBStateBackend;
 import org.apache.flink.core.fs.Path;
 import org.apache.flink.runtime.checkpoint.OperatorSubtaskState;
 import org.apache.flink.state.api.functions.KeyedStateBootstrapFunction;
 import org.apache.flink.state.api.output.operators.KeyedStateBootstrapOperator;
+import org.apache.flink.streaming.api.TimeDomain;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.streaming.api.operators.KeyedProcessOperator;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.api.operators.StreamMap;
 import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness;
+import org.apache.flink.util.Collector;
 
 import org.hamcrest.Matchers;
 import org.junit.Assert;
@@ -43,52 +48,119 @@ import org.junit.rules.TemporaryFolder;
  */
 public class KeyedStateBootstrapOperatorTest {
 
-	private static final ValueStateDescriptor<Long> descriptor = new ValueStateDescriptor<Long>("state", Types.LONG);
+	private static final ValueStateDescriptor<Long> descriptor = new ValueStateDescriptor<>("state", Types.LONG);
+
+	private static final Long EVENT_TIMER = Long.MAX_VALUE - 1;
+
+	private static final Long PROC_TIMER = Long.MAX_VALUE - 2;
 
 	@Rule
 	public TemporaryFolder folder = new TemporaryFolder();
 
 	@Test
-	public void testNonTimerStatesRestorableByNonProcessesOperator() throws Exception {
+	public void testTimerStateRestorable() throws Exception {
 		Path path = new Path(folder.newFolder().toURI());
 
 		OperatorSubtaskState state;
-		KeyedStateBootstrapOperator<Long, Long> bootstrapOperator = new KeyedStateBootstrapOperator<>(0L, path, new SimpleBootstrapFunction());
+		KeyedStateBootstrapOperator<Long, Long> bootstrapOperator = new KeyedStateBootstrapOperator<>(0L, path, new TimerBootstrapFunction());
 		try (KeyedOneInputStreamOperatorTestHarness<Long, Long, TaggedOperatorSubtaskState> harness = getHarness(bootstrapOperator)) {
-			harness.open();
-
-			harness.processElement(1L, 0L);
-			harness.processElement(2L, 0L);
-			harness.processElement(3L, 0L);
-			bootstrapOperator.endInput();
-
-			state = harness.extractOutputValues().get(0).state;
+			processElements(harness, 1L, 2L, 3L);
+			state = getState(bootstrapOperator, harness);
 		}
 
-		StreamMap<Long, Long> mapOperator = new StreamMap<>(new StreamingFunction());
-		try (KeyedOneInputStreamOperatorTestHarness<Long, Long, Long> harness = getHarness(mapOperator)) {
+		KeyedProcessOperator<Long, Long, Tuple3<Long, Long, TimeDomain>> procOperator = new KeyedProcessOperator<>(new SimpleProcessFunction());
+		try (KeyedOneInputStreamOperatorTestHarness<Long, Long, Tuple3<Long, Long, TimeDomain>> harness = getHarness(procOperator, state)) {
+			harness.processWatermark(EVENT_TIMER);
+			harness.setProcessingTime(PROC_TIMER);
+
+			assertHarnessOutput(harness,
+				Tuple3.of(1L, EVENT_TIMER, TimeDomain.EVENT_TIME),
+				Tuple3.of(2L, EVENT_TIMER, TimeDomain.EVENT_TIME),
+				Tuple3.of(3L, EVENT_TIMER, TimeDomain.EVENT_TIME),
+				Tuple3.of(1L, PROC_TIMER, TimeDomain.PROCESSING_TIME),
+				Tuple3.of(2L, PROC_TIMER, TimeDomain.PROCESSING_TIME),
+				Tuple3.of(3L, PROC_TIMER, TimeDomain.PROCESSING_TIME));
+		}
+	}
 
-			harness.initializeState(state);
-			harness.open();
+	@Test
+	public void testNonTimerStatesRestorableByNonProcessesOperator() throws Exception {
+		Path path = new Path(folder.newFolder().toURI());
 
-			harness.processElement(1L, 0L);
-			harness.processElement(2L, 0L);
-			harness.processElement(3L, 0L);
+		OperatorSubtaskState state;
+		KeyedStateBootstrapOperator<Long, Long> bootstrapOperator = new KeyedStateBootstrapOperator<>(0L, path, new SimpleBootstrapFunction());
+		try (KeyedOneInputStreamOperatorTestHarness<Long, Long, TaggedOperatorSubtaskState> harness = getHarness(bootstrapOperator)) {
+			processElements(harness, 1L, 2L, 3L);
+			state = getState(bootstrapOperator, harness);
+		}
 
-			Assert.assertThat(harness.extractOutputValues(), Matchers.containsInAnyOrder(1L, 2L, 3L));
+		StreamMap<Long, Long> mapOperator = new StreamMap<>(new StreamingFunction());
+		try (KeyedOneInputStreamOperatorTestHarness<Long, Long, Long> harness = getHarness(mapOperator, state)) {
+			processElements(harness, 1L, 2L, 3L);
 
+			assertHarnessOutput(harness, 1L, 2L, 3L);
 			harness.snapshot(0L, 0L);
 		}
 	}
 
 	private <T> KeyedOneInputStreamOperatorTestHarness<Long, Long, T> getHarness(OneInputStreamOperator<Long, T> bootstrapOperator) throws Exception {
+		return getHarness(bootstrapOperator, null);
+	}
+
+	private <T> KeyedOneInputStreamOperatorTestHarness<Long, Long, T> getHarness(OneInputStreamOperator<Long, T> bootstrapOperator, OperatorSubtaskState state) throws Exception {
 		KeyedOneInputStreamOperatorTestHarness<Long, Long, T> harness = new KeyedOneInputStreamOperatorTestHarness<>(
 			bootstrapOperator, id -> id, Types.LONG, 128, 1, 0);
 
 		harness.setStateBackend(new RocksDBStateBackend(folder.newFolder().toURI()));
+		if (state != null) {
+			harness.initializeState(state);
+		}
+		harness.open();
 		return harness;
 	}
 
+	private <T> void processElements(KeyedOneInputStreamOperatorTestHarness<Long, Long, T> harness, Long... records) throws Exception {
+		for (Long record : records) {
+			harness.processElement(record, 0);
+		}
+	}
+
+	private OperatorSubtaskState getState(
+		KeyedStateBootstrapOperator<Long, Long> bootstrapOperator,
+		KeyedOneInputStreamOperatorTestHarness<Long, Long, TaggedOperatorSubtaskState> harness) throws Exception {
+		OperatorSubtaskState state;
+		bootstrapOperator.endInput();
+		state = harness.extractOutputValues().get(0).state;
+		return state;
+	}
+
+	private <T> void assertHarnessOutput(KeyedOneInputStreamOperatorTestHarness<Long, Long, T> harness, T... output) {
+		Assert.assertThat("The output from the operator does not match the expected values",
+			harness.extractOutputValues(), Matchers.containsInAnyOrder(output));
+	}
+
+	private static class TimerBootstrapFunction extends KeyedStateBootstrapFunction<Long, Long> {
+
+		@Override
+		public void processElement(Long value, Context ctx) throws Exception {
+			ctx.timerService().registerEventTimeTimer(EVENT_TIMER);
+			ctx.timerService().registerProcessingTimeTimer(PROC_TIMER);
+		}
+	}
+
+	private static class SimpleProcessFunction extends KeyedProcessFunction<Long, Long, Tuple3<Long, Long, TimeDomain>> {
+
+		@Override
+		public void processElement(Long value, Context ctx, Collector<Tuple3<Long, Long, TimeDomain>> out) throws Exception {
+
+		}
+
+		@Override
+		public void onTimer(long timestamp, OnTimerContext ctx, Collector<Tuple3<Long, Long, TimeDomain>> out) throws Exception {
+			out.collect(Tuple3.of(ctx.getCurrentKey(), timestamp, ctx.timeDomain()));
+		}
+	}
+
 	private static class SimpleBootstrapFunction extends KeyedStateBootstrapFunction<Long, Long> {
 
 		private ValueState<Long> state;