You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2021/09/03 10:27:19 UTC

[flink] branch master updated (0a2607c -> 36f607c)

This is an automated email from the ASF dual-hosted git repository.

dwysakowicz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from 0a2607c  [FLINK-23654][runtime] Splitting IO and future threadpools and adding advanced configurations
     new f6e7847  [FLINK-21090] Add tests for stop-with-savepoint and FLIP-147
     new 36f607c  [FLINK-21090] Minor code improvements

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../flink/runtime/testutils/CommonTestUtils.java   |  73 ++++-
 .../operators/lifecycle/BoundedSourceITCase.java   |  78 ++++++
 .../lifecycle/PartiallyFinishedSourcesITCase.java  | 145 ++++++++++
 .../lifecycle/StopWithSavepointITCase.java         | 155 +++++++++++
 .../operators/lifecycle/TestJobExecutor.java       | 223 ++++++++++++++++
 .../lifecycle/TestJobWithDescription.java          |  57 ++++
 .../command/SharedTestCommandDispatcher.java       |  48 ++++
 .../operators/lifecycle/command/TestCommand.java   |  51 ++++
 .../lifecycle/command/TestCommandDispatcher.java   |  50 ++++
 .../command/TestCommandDispatcherImpl.java         |  68 +++++
 .../lifecycle/event/CheckpointCompletedEvent.java  |  62 +++++
 .../lifecycle/event/CheckpointStartedEvent.java    |  63 +++++
 .../operators/lifecycle/event/InputEndedEvent.java |  63 +++++
 .../lifecycle/event/OperatorFinishedEvent.java     |  70 +++++
 .../lifecycle/event/OperatorStartedEvent.java      |  37 +++
 .../lifecycle/event/SharedTestEventQueue.java      |  60 +++++
 .../lifecycle/event/TestCommandAckEvent.java       |  46 ++++
 .../operators/lifecycle/event/TestEvent.java       |  63 +++++
 .../operators/lifecycle/event/TestEventQueue.java  |  55 ++++
 .../lifecycle/event/TestEventQueueImpl.java        |  66 +++++
 .../lifecycle/event/WatermarkReceivedEvent.java    |  65 +++++
 .../lifecycle/graph/MultiInputTestOperator.java    | 209 +++++++++++++++
 .../graph/MultiInputTestOperatorFactory.java       |  60 +++++
 .../graph/OneInputTestStreamOperator.java          | 144 ++++++++++
 .../graph/OneInputTestStreamOperatorFactory.java   |  74 ++++++
 .../operators/lifecycle/graph/TestDataElement.java |  61 +++++
 .../operators/lifecycle/graph/TestEventSource.java | 119 +++++++++
 .../operators/lifecycle/graph/TestJobBuilders.java | 295 +++++++++++++++++++++
 .../graph/TwoInputTestStreamOperator.java          | 171 ++++++++++++
 .../lifecycle/validation/DrainingValidator.java    |  97 +++++++
 .../lifecycle/validation/FinishingValidator.java   |  86 ++++++
 .../validation/SameCheckpointValidator.java        |  81 ++++++
 .../validation/TestJobDataFlowValidator.java       | 153 +++++++++++
 .../validation/TestOperatorLifecycleValidator.java |  58 ++++
 34 files changed, 3196 insertions(+), 10 deletions(-)
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/TestJobExecutor.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/TestJobWithDescription.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/SharedTestCommandDispatcher.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommand.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommandDispatcher.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommandDispatcherImpl.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/CheckpointCompletedEvent.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/CheckpointStartedEvent.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/InputEndedEvent.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/OperatorFinishedEvent.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/OperatorStartedEvent.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/SharedTestEventQueue.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestCommandAckEvent.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEvent.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEventQueue.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEventQueueImpl.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/WatermarkReceivedEvent.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/MultiInputTestOperator.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/MultiInputTestOperatorFactory.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/OneInputTestStreamOperator.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/OneInputTestStreamOperatorFactory.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestDataElement.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestEventSource.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestJobBuilders.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TwoInputTestStreamOperator.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/DrainingValidator.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/FinishingValidator.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/SameCheckpointValidator.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/TestJobDataFlowValidator.java
 create mode 100644 flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/TestOperatorLifecycleValidator.java

[flink] 02/02: [FLINK-21090] Minor code improvements

Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 36f607c9fdff11e67cd1768bb48295f83da3e350
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Sep 2 13:07:45 2021 +0200

    [FLINK-21090] Minor code improvements
    
    * Do not extend from the TriFunction
    * Use non deprecated watermark assigners
    * Log exceptions in case of a global termination state
    
    This closes #16773
---
 .../flink/runtime/testutils/CommonTestUtils.java   | 23 +++++---
 .../operators/lifecycle/BoundedSourceITCase.java   |  2 +-
 .../lifecycle/PartiallyFinishedSourcesITCase.java  |  2 +-
 .../lifecycle/StopWithSavepointITCase.java         |  2 +-
 .../operators/lifecycle/graph/TestJobBuilders.java | 69 +++++++++++-----------
 5 files changed, 54 insertions(+), 44 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
index 91ff6aa..7af6725 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
@@ -25,10 +25,10 @@ import org.apache.flink.core.execution.JobClient;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
 import org.apache.flink.runtime.executiongraph.AccessExecutionVertex;
+import org.apache.flink.runtime.executiongraph.ErrorInfo;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.minicluster.MiniCluster;
 import org.apache.flink.util.FileUtils;
-import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.function.SupplierWithException;
 
 import java.io.BufferedInputStream;
@@ -49,6 +49,9 @@ import java.util.concurrent.TimeoutException;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 
+import static java.lang.String.format;
+import static org.junit.jupiter.api.Assertions.fail;
+
 /** This class contains auxiliary methods for unit tests. */
 public class CommonTestUtils {
 
@@ -205,10 +208,14 @@ public class CommonTestUtils {
         waitUntilCondition(
                 () -> {
                     final AccessExecutionGraph graph = executionGraphSupplier.get();
-                    Preconditions.checkState(
-                            !graph.getState().isGloballyTerminalState(),
-                            "Graph is in globally terminal state (%s)",
-                            graph.getState());
+                    if (graph.getState().isGloballyTerminalState()) {
+                        final ErrorInfo failureInfo = graph.getFailureInfo();
+                        fail(
+                                format(
+                                        "Graph is in globally terminal state (%s)",
+                                        graph.getState()),
+                                failureInfo != null ? failureInfo.getException() : null);
+                    }
                     return graph.getState() == JobStatus.RUNNING
                             && graph.getAllVertices().values().stream()
                                     .allMatch(
@@ -249,13 +256,13 @@ public class CommonTestUtils {
                             client.getJobExecutionResult().get();
                         } catch (Exception e) {
                             throw new IllegalStateException(
-                                    String.format(
+                                    format(
                                             "Job has entered %s state, but expecting %s",
                                             currentStatus, expectedStatus),
                                     e);
                         }
                         throw new IllegalStateException(
-                                String.format(
+                                format(
                                         "Job has entered a terminal state %s, but expecting %s",
                                         currentStatus, expectedStatus));
                     }
@@ -280,7 +287,7 @@ public class CommonTestUtils {
                         return true;
                     } else if (state.isTerminal()) {
                         throw new RuntimeException(
-                                String.format(
+                                format(
                                         "Sub-Task %s is already in a terminal state %s",
                                         subtask, state));
                     } else {
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
index f222bc1..6234ba2 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
@@ -61,7 +61,7 @@ public class BoundedSourceITCase extends AbstractTestBase {
     @Test
     public void test() throws Exception {
         TestJobWithDescription testJob =
-                graphBuilder.apply(
+                graphBuilder.build(
                         sharedObjects,
                         cfg -> cfg.setBoolean(ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true),
                         env -> {});
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
index b4f1bb1..84054e5 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
@@ -96,7 +96,7 @@ public class PartiallyFinishedSourcesITCase extends AbstractTestBase {
     }
 
     private TestJobWithDescription buildJob() {
-        return graphBuilder.apply(
+        return graphBuilder.build(
                 sharedObjects,
                 cfg -> cfg.setBoolean(ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true),
                 env -> {
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
index 5539fe9..ea100b1 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
@@ -108,7 +108,7 @@ public class StopWithSavepointITCase extends AbstractTestBase {
 
     @Test
     public void test() throws Exception {
-        TestJobWithDescription testJob = graphBuilder.apply(sharedObjects, cfg -> {}, env -> {});
+        TestJobWithDescription testJob = graphBuilder.build(sharedObjects, cfg -> {}, env -> {});
 
         TestJobExecutor.execute(testJob, miniClusterResource)
                 .waitForEvent(WatermarkReceivedEvent.class)
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestJobBuilders.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestJobBuilders.java
index eb2ab1a..4afeec7 100644
--- a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestJobBuilders.java
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestJobBuilders.java
@@ -17,6 +17,11 @@
 
 package org.apache.flink.runtime.operators.lifecycle.graph;
 
+import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
+import org.apache.flink.api.common.eventtime.Watermark;
+import org.apache.flink.api.common.eventtime.WatermarkGenerator;
+import org.apache.flink.api.common.eventtime.WatermarkOutput;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription;
@@ -27,15 +32,10 @@ import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.MultipleConnectedStreams;
 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
 import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
 import org.apache.flink.streaming.api.operators.ChainingStrategy;
 import org.apache.flink.streaming.api.transformations.MultipleInputTransformation;
-import org.apache.flink.streaming.api.watermark.Watermark;
 import org.apache.flink.testutils.junit.SharedObjects;
-import org.apache.flink.util.function.TriFunction;
-
-import javax.annotation.Nullable;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -52,19 +52,20 @@ import static org.apache.flink.configuration.JobManagerOptions.EXECUTION_FAILOVE
 public class TestJobBuilders {
 
     /** {@link TestJobWithDescription} builder. */
-    public interface TestingGraphBuilder
-            extends TriFunction<
-                    SharedObjects,
-                    Consumer<Configuration>,
-                    Consumer<StreamExecutionEnvironment>,
-                    TestJobWithDescription> {}
+    @FunctionalInterface
+    public interface TestingGraphBuilder {
+        TestJobWithDescription build(
+                SharedObjects shared,
+                Consumer<Configuration> modifyConfig,
+                Consumer<StreamExecutionEnvironment> modifyEnvironment);
+    }
 
     private TestJobBuilders() {}
 
     public static final TestingGraphBuilder SIMPLE_GRAPH_BUILDER =
             new TestingGraphBuilder() {
                 @Override
-                public TestJobWithDescription apply(
+                public TestJobWithDescription build(
                         SharedObjects shared,
                         Consumer<Configuration> confConsumer,
                         Consumer<StreamExecutionEnvironment> envConsumer) {
@@ -116,7 +117,7 @@ public class TestJobBuilders {
     public static final TestingGraphBuilder COMPLEX_GRAPH_BUILDER =
             new TestingGraphBuilder() {
                 @Override
-                public TestJobWithDescription apply(
+                public TestJobWithDescription build(
                         SharedObjects shared,
                         Consumer<Configuration> confConsumer,
                         Consumer<StreamExecutionEnvironment> envConsumer) {
@@ -268,25 +269,27 @@ public class TestJobBuilders {
 
     private static final String OP_ID_HASH_PREFIX = "0000000000000000000000000000000";
 
-    @SuppressWarnings("deprecation")
-    private static AssignerWithPeriodicWatermarks<TestDataElement> createWmAssigner() {
-        return new AssignerWithPeriodicWatermarks<TestDataElement>() {
-            private Watermark watermark;
-
-            @Nullable
-            @Override
-            public Watermark getCurrentWatermark() {
-                return watermark;
-            }
-
-            @Override
-            public long extractTimestamp(TestDataElement element, long recordTimestamp) {
-                if (element instanceof TestDataElement) {
-                    this.watermark = new Watermark(((TestDataElement) element).seq);
-                    return ((TestDataElement) element).seq;
-                }
-                return recordTimestamp;
-            }
-        };
+    private static WatermarkStrategy<TestDataElement> createWmAssigner() {
+        return WatermarkStrategy.forGenerator(
+                        ctx ->
+                                new WatermarkGenerator<TestDataElement>() {
+                                    private Watermark watermark = new Watermark(Long.MIN_VALUE);
+
+                                    @Override
+                                    public void onEvent(
+                                            TestDataElement event,
+                                            long eventTimestamp,
+                                            WatermarkOutput output) {
+                                        this.watermark = new Watermark(eventTimestamp);
+                                    }
+
+                                    @Override
+                                    public void onPeriodicEmit(WatermarkOutput output) {
+                                        output.emitWatermark(watermark);
+                                    }
+                                })
+                .withTimestampAssigner(
+                        (SerializableTimestampAssigner<TestDataElement>)
+                                (element, recordTimestamp) -> element.seq);
     }
 }

[flink] 01/02: [FLINK-21090] Add tests for stop-with-savepoint and FLIP-147

Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit f6e784772dc89e197efe6d7a5f52a513ced6682a
Author: Roman Khachatryan <kh...@gmail.com>
AuthorDate: Tue Aug 10 15:10:33 2021 +0100

    [FLINK-21090] Add tests for stop-with-savepoint and FLIP-147
---
 .../flink/runtime/testutils/CommonTestUtils.java   |  52 +++-
 .../operators/lifecycle/BoundedSourceITCase.java   |  78 ++++++
 .../lifecycle/PartiallyFinishedSourcesITCase.java  | 145 ++++++++++
 .../lifecycle/StopWithSavepointITCase.java         | 155 +++++++++++
 .../operators/lifecycle/TestJobExecutor.java       | 223 ++++++++++++++++
 .../lifecycle/TestJobWithDescription.java          |  57 ++++
 .../command/SharedTestCommandDispatcher.java       |  48 ++++
 .../operators/lifecycle/command/TestCommand.java   |  51 ++++
 .../lifecycle/command/TestCommandDispatcher.java   |  50 ++++
 .../command/TestCommandDispatcherImpl.java         |  68 +++++
 .../lifecycle/event/CheckpointCompletedEvent.java  |  62 +++++
 .../lifecycle/event/CheckpointStartedEvent.java    |  63 +++++
 .../operators/lifecycle/event/InputEndedEvent.java |  63 +++++
 .../lifecycle/event/OperatorFinishedEvent.java     |  70 +++++
 .../lifecycle/event/OperatorStartedEvent.java      |  37 +++
 .../lifecycle/event/SharedTestEventQueue.java      |  60 +++++
 .../lifecycle/event/TestCommandAckEvent.java       |  46 ++++
 .../operators/lifecycle/event/TestEvent.java       |  63 +++++
 .../operators/lifecycle/event/TestEventQueue.java  |  55 ++++
 .../lifecycle/event/TestEventQueueImpl.java        |  66 +++++
 .../lifecycle/event/WatermarkReceivedEvent.java    |  65 +++++
 .../lifecycle/graph/MultiInputTestOperator.java    | 209 +++++++++++++++
 .../graph/MultiInputTestOperatorFactory.java       |  60 +++++
 .../graph/OneInputTestStreamOperator.java          | 144 ++++++++++
 .../graph/OneInputTestStreamOperatorFactory.java   |  74 ++++++
 .../operators/lifecycle/graph/TestDataElement.java |  61 +++++
 .../operators/lifecycle/graph/TestEventSource.java | 119 +++++++++
 .../operators/lifecycle/graph/TestJobBuilders.java | 292 +++++++++++++++++++++
 .../graph/TwoInputTestStreamOperator.java          | 171 ++++++++++++
 .../lifecycle/validation/DrainingValidator.java    |  97 +++++++
 .../lifecycle/validation/FinishingValidator.java   |  86 ++++++
 .../validation/SameCheckpointValidator.java        |  81 ++++++
 .../validation/TestJobDataFlowValidator.java       | 153 +++++++++++
 .../validation/TestOperatorLifecycleValidator.java |  58 ++++
 34 files changed, 3179 insertions(+), 3 deletions(-)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
index b66ee2a..91ff6aa 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/CommonTestUtils.java
@@ -22,8 +22,10 @@ import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.JobStatus;
 import org.apache.flink.api.common.time.Deadline;
 import org.apache.flink.core.execution.JobClient;
+import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
 import org.apache.flink.runtime.executiongraph.AccessExecutionVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
 import org.apache.flink.runtime.minicluster.MiniCluster;
 import org.apache.flink.util.FileUtils;
 import org.apache.flink.util.Preconditions;
@@ -45,6 +47,7 @@ import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.function.Predicate;
+import java.util.stream.Stream;
 
 /** This class contains auxiliary methods for unit tests. */
 public class CommonTestUtils {
@@ -161,9 +164,12 @@ public class CommonTestUtils {
 
     public static void waitForAllTaskRunning(
             MiniCluster miniCluster, JobID jobId, boolean allowFinished) throws Exception {
-        waitForAllTaskRunning(
-                () -> miniCluster.getExecutionGraph(jobId).get(60, TimeUnit.SECONDS),
-                allowFinished);
+        waitForAllTaskRunning(() -> getGraph(miniCluster, jobId), allowFinished);
+    }
+
+    private static AccessExecutionGraph getGraph(MiniCluster miniCluster, JobID jobId)
+            throws InterruptedException, java.util.concurrent.ExecutionException, TimeoutException {
+        return miniCluster.getExecutionGraph(jobId).get(60, TimeUnit.SECONDS);
     }
 
     public static void waitForAllTaskRunning(
@@ -264,6 +270,46 @@ public class CommonTestUtils {
         client.cancel().get(timeout.toMillis(), TimeUnit.MILLISECONDS);
     }
 
+    public static void waitForSubtasksToFinish(
+            MiniCluster miniCluster, JobID job, JobVertexID id, boolean allSubtasks)
+            throws Exception {
+        Predicate<AccessExecutionVertex> subtaskPredicate =
+                subtask -> {
+                    ExecutionState state = subtask.getExecutionState();
+                    if (state == ExecutionState.FINISHED) {
+                        return true;
+                    } else if (state.isTerminal()) {
+                        throw new RuntimeException(
+                                String.format(
+                                        "Sub-Task %s is already in a terminal state %s",
+                                        subtask, state));
+                    } else {
+                        return false;
+                    }
+                };
+        waitUntilCondition(
+                () -> {
+                    AccessExecutionGraph graph = getGraph(miniCluster, job);
+                    if (graph.getState() != JobStatus.RUNNING) {
+                        return false;
+                    }
+                    Stream<AccessExecutionVertex> vertexStream =
+                            Arrays.stream(
+                                    graph.getAllVertices().values().stream()
+                                            .filter(jv -> jv.getJobVertexId().equals(id))
+                                            .findAny()
+                                            .orElseThrow(
+                                                    () ->
+                                                            new RuntimeException(
+                                                                    "Vertex not found " + id))
+                                            .getTaskVertices());
+                    return allSubtasks
+                            ? vertexStream.allMatch(subtaskPredicate)
+                            : vertexStream.anyMatch(subtaskPredicate);
+                },
+                Deadline.fromNow(Duration.of(1, ChronoUnit.MINUTES)));
+    }
+
     /** Utility class to read the output of a process stream and forward it into a StringWriter. */
     public static class PipeForwarder extends Thread {
 
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
new file mode 100644
index 0000000..f222bc1
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/BoundedSourceITCase.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle;
+
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent;
+import org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.TestingGraphBuilder;
+import org.apache.flink.runtime.operators.lifecycle.validation.DrainingValidator;
+import org.apache.flink.runtime.operators.lifecycle.validation.FinishingValidator;
+import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.testutils.junit.SharedObjects;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommand.FINISH_SOURCES;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher.TestCommandScope.ALL_SUBTASKS;
+import static org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.COMPLEX_GRAPH_BUILDER;
+import static org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.SIMPLE_GRAPH_BUILDER;
+import static org.apache.flink.runtime.operators.lifecycle.validation.TestJobDataFlowValidator.checkDataFlow;
+import static org.apache.flink.runtime.operators.lifecycle.validation.TestOperatorLifecycleValidator.checkOperatorsLifecycle;
+import static org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH;
+
+/**
+ * A test suite to check that the operator methods are called according to contract when sources are
+ * finishing normally. The contract was refined in FLIP-147.
+ *
+ * <p>The checks are similar to those in {@link StopWithSavepointITCase} {@link
+ * StopWithSavepointITCase#withDrain withDrain} except that final checkpoint doesn't have to be the
+ * same.
+ */
+@RunWith(Parameterized.class)
+public class BoundedSourceITCase extends AbstractTestBase {
+
+    @Rule public final SharedObjects sharedObjects = SharedObjects.create();
+
+    @Parameter public TestingGraphBuilder graphBuilder;
+
+    @Parameterized.Parameters(name = "{0}")
+    public static Object[] parameters() {
+        return new Object[] {SIMPLE_GRAPH_BUILDER, COMPLEX_GRAPH_BUILDER};
+    }
+
+    @Test
+    public void test() throws Exception {
+        TestJobWithDescription testJob =
+                graphBuilder.apply(
+                        sharedObjects,
+                        cfg -> cfg.setBoolean(ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true),
+                        env -> {});
+
+        TestJobExecutor.execute(testJob, miniClusterResource)
+                .waitForEvent(CheckpointCompletedEvent.class)
+                .sendBroadcastCommand(FINISH_SOURCES, ALL_SUBTASKS)
+                .waitForTermination()
+                .assertFinishedSuccessfully();
+
+        checkOperatorsLifecycle(testJob, new DrainingValidator(), new FinishingValidator());
+        checkDataFlow(testJob);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
new file mode 100644
index 0000000..b4f1bb1
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/PartiallyFinishedSourcesITCase.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle;
+
+import org.apache.flink.runtime.OperatorIDPair;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher.TestCommandScope;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent;
+import org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.TestingGraphBuilder;
+import org.apache.flink.runtime.operators.lifecycle.validation.DrainingValidator;
+import org.apache.flink.runtime.operators.lifecycle.validation.FinishingValidator;
+import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.testutils.junit.SharedObjects;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+
+import static java.util.stream.StreamSupport.stream;
+import static org.apache.flink.api.common.restartstrategy.RestartStrategies.fixedDelayRestart;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommand.FINISH_SOURCES;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher.TestCommandScope.ALL_SUBTASKS;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher.TestCommandScope.SINGLE_SUBTASK;
+import static org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.COMPLEX_GRAPH_BUILDER;
+import static org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.SIMPLE_GRAPH_BUILDER;
+import static org.apache.flink.runtime.operators.lifecycle.validation.TestJobDataFlowValidator.checkDataFlow;
+import static org.apache.flink.runtime.operators.lifecycle.validation.TestOperatorLifecycleValidator.checkOperatorsLifecycle;
+import static org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH;
+
+/**
+ * A test suite to check that the operator methods are called according to contract when sources are
+ * finishing partially; i.e. either some subtasks of a source have finished; or all subtasks of a
+ * single source have finished. The feature was implemented in FLIP-147.
+ *
+ * <p>The checks are similar to those in {@link StopWithSavepointITCase} {@link
+ * StopWithSavepointITCase#withDrain withDrain} except that the final checkpoint doesn't have to be
+ * the same.
+ */
+@RunWith(Parameterized.class)
+public class PartiallyFinishedSourcesITCase extends AbstractTestBase {
+
+    @Rule public final SharedObjects sharedObjects = SharedObjects.create();
+
+    @Parameter(0)
+    public TestingGraphBuilder graphBuilder;
+
+    @Parameter(1)
+    public TestCommandScope subtaskScope;
+
+    @Parameter(2)
+    public boolean failover;
+
+    @Test
+    public void test() throws Exception {
+        TestJobWithDescription testJob = buildJob();
+
+        // pick any source operator
+        String finishingOperatorID = testJob.sources.iterator().next();
+        JobVertexID finishingVertexID = findJobVertexID(testJob, finishingOperatorID);
+
+        TestJobExecutor executor =
+                TestJobExecutor.execute(testJob, miniClusterResource)
+                        .waitForEvent(CheckpointCompletedEvent.class)
+                        .sendOperatorCommand(finishingOperatorID, FINISH_SOURCES, subtaskScope)
+                        .waitForSubtasksToFinish(finishingVertexID, subtaskScope)
+                        // wait for a checkpoint to complete with a finished subtask(s)
+                        // but skip one checkpoint that might be started before finishing
+                        .waitForEvent(CheckpointCompletedEvent.class)
+                        .waitForEvent(CheckpointCompletedEvent.class);
+        if (failover) {
+            executor.triggerFailover();
+        }
+        executor.sendBroadcastCommand(FINISH_SOURCES, ALL_SUBTASKS)
+                .waitForTermination()
+                .assertFinishedSuccessfully();
+
+        checkOperatorsLifecycle(testJob, new DrainingValidator(), new FinishingValidator());
+        checkDataFlow(testJob);
+    }
+
+    private TestJobWithDescription buildJob() {
+        return graphBuilder.apply(
+                sharedObjects,
+                cfg -> cfg.setBoolean(ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true),
+                env -> {
+                    env.setRestartStrategy(fixedDelayRestart(1, 0));
+                    // checkpoints can hang (because of not yet fixed bugs and triggering
+                    // checkpoint while the source finishes), so let them timeout quickly
+                    env.getCheckpointConfig().setCheckpointTimeout(5000);
+                    // but don't fail the job
+                    env.getCheckpointConfig()
+                            .setTolerableCheckpointFailureNumber(Integer.MAX_VALUE);
+                    // explicitly set to one to ease avoiding race conditions
+                    env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
+                });
+    }
+
+    private JobVertexID findJobVertexID(
+            TestJobWithDescription testJob, String finishingOperatorID) {
+        return stream(testJob.jobGraph.getVertices().spliterator(), false)
+                .filter(
+                        v ->
+                                v.getOperatorIDs().stream()
+                                        .anyMatch(idPair -> matches(idPair, finishingOperatorID)))
+                .findAny()
+                .orElseThrow(() -> new RuntimeException("Vertex not found: " + finishingOperatorID))
+                .getID();
+    }
+
+    private boolean matches(OperatorIDPair idPair, String operatorID) {
+        return idPair.getUserDefinedOperatorID()
+                .orElse(idPair.getGeneratedOperatorID())
+                .toString()
+                .equals(operatorID);
+    }
+
+    @Parameterized.Parameters(name = "{0} {1}, failover: {2}")
+    public static Object[] parameters() {
+        return new Object[] {
+            new Object[] {SIMPLE_GRAPH_BUILDER, SINGLE_SUBTASK, true},
+            new Object[] {COMPLEX_GRAPH_BUILDER, SINGLE_SUBTASK, true},
+            new Object[] {COMPLEX_GRAPH_BUILDER, ALL_SUBTASKS, true},
+            new Object[] {SIMPLE_GRAPH_BUILDER, SINGLE_SUBTASK, false},
+            new Object[] {COMPLEX_GRAPH_BUILDER, SINGLE_SUBTASK, false},
+            new Object[] {COMPLEX_GRAPH_BUILDER, ALL_SUBTASKS, false},
+        };
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
new file mode 100644
index 0000000..5539fe9
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/StopWithSavepointITCase.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle;
+
+import org.apache.flink.api.common.state.CheckpointListener;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent;
+import org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.TestingGraphBuilder;
+import org.apache.flink.runtime.operators.lifecycle.validation.DrainingValidator;
+import org.apache.flink.runtime.operators.lifecycle.validation.FinishingValidator;
+import org.apache.flink.runtime.operators.lifecycle.validation.SameCheckpointValidator;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.BoundedMultiInput;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.testutils.junit.SharedObjects;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+
+import java.util.List;
+
+import static org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.COMPLEX_GRAPH_BUILDER;
+import static org.apache.flink.runtime.operators.lifecycle.graph.TestJobBuilders.SIMPLE_GRAPH_BUILDER;
+import static org.apache.flink.runtime.operators.lifecycle.validation.TestJobDataFlowValidator.checkDataFlow;
+import static org.apache.flink.runtime.operators.lifecycle.validation.TestOperatorLifecycleValidator.checkOperatorsLifecycle;
+
+/**
+ * A test suite to check that the operator methods are called according to contract when the job is
+ * stopped with savepoint. The contract was refined in FLIP-147.
+ *
+ * <p>Checked assumptions:
+ *
+ * <ol>
+ *   <li>Downstream should only be "finished" after all of its the upstreams are
+ *   <li>Order of events when finishing an operator:
+ *       <ol>
+ *         <li>(last data element)
+ *         <li>{@link Watermark#MAX_WATERMARK MAX_WATERMARK} (if with drain)
+ *         <li>{@link BoundedMultiInput#endInput endInput} (if with drain)
+ *         <li>timer service quiesced
+ *         <li>{@link StreamOperator#finish() finish} (if with drain; support is planned for
+ *             no-drain)
+ *         <li>{@link AbstractStreamOperator#snapshotState(StateSnapshotContext) snapshotState} (for
+ *             the respective checkpoint)
+ *         <li>{@link CheckpointListener#notifyCheckpointComplete notifyCheckpointComplete} (for the
+ *             respective checkpoint)
+ *         <li>(task termination)
+ *       </ol>
+ *   <li>Timers can be registered until the operator is finished (though may not fire) (simply
+ *       register every 1ms and don't expect any exception)
+ *   <li>The same watermark is received
+ * </ol>
+ *
+ * <p>Variants:
+ *
+ * <ul>
+ *   <li>command - with or without drain (MAX_WATERMARK and endInput should be iff drain)
+ *   <li>graph - different exchanges (keyBy, forward)
+ *   <li>graph - multi-inputs, unions
+ *   <li>graph - FLIP-27 and regular sources (should work for both) - FLIP-27 not implemented
+ * </ul>
+ *
+ * <p>Not checked:
+ *
+ * <ul>
+ *   <li>state distribution on recovery (when a new job started from the taken savepoint) (a
+ *       separate IT case for partial finishing and state distribution)
+ *   <li>re-taking a savepoint after one fails (and job fails over) (as it should not affect
+ *       savepoints)
+ *   <li>taking a savepoint after recovery (as it should not affect savepoints)
+ *   <li>taking a savepoint on a partially completed graph (a separate IT case)
+ * </ul>
+ */
+@RunWith(Parameterized.class)
+public class StopWithSavepointITCase extends AbstractTestBase {
+
+    @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+    @Rule public final SharedObjects sharedObjects = SharedObjects.create();
+
+    @Parameter(0)
+    public boolean withDrain;
+
+    @Parameter(1)
+    public TestingGraphBuilder graphBuilder;
+
+    @Test
+    public void test() throws Exception {
+        TestJobWithDescription testJob = graphBuilder.apply(sharedObjects, cfg -> {}, env -> {});
+
+        TestJobExecutor.execute(testJob, miniClusterResource)
+                .waitForEvent(WatermarkReceivedEvent.class)
+                .stopWithSavepoint(temporaryFolder, withDrain);
+
+        SameCheckpointValidator sameCheckpointValidator =
+                // note: using highest checkpoint will not work with partially finishing tasks
+                // in that case, savepoint ID can be inferred from savepoint path
+                new SameCheckpointValidator(getHighestCheckpoint(testJob.eventQueue.getAll()));
+
+        if (withDrain) {
+            checkOperatorsLifecycle(
+                    testJob,
+                    sameCheckpointValidator,
+                    new DrainingValidator(),
+                    /* Currently (1.14), finish is only called with drain; todo: enable after updating production code */
+                    new FinishingValidator());
+        } else {
+            checkOperatorsLifecycle(testJob, sameCheckpointValidator);
+        }
+
+        if (withDrain) {
+            // currently (1.14), sources do not stop before taking a savepoint and continue emission
+            // todo: enable after updating production code
+            checkDataFlow(testJob);
+        }
+    }
+
+    @Parameterized.Parameters(name = "withDrain: {0}, {1}")
+    public static Object[] parameters() {
+        return new Object[][] {
+            new Object[] {true, SIMPLE_GRAPH_BUILDER}, new Object[] {false, SIMPLE_GRAPH_BUILDER},
+            new Object[] {true, COMPLEX_GRAPH_BUILDER}, new Object[] {false, COMPLEX_GRAPH_BUILDER},
+        };
+    }
+
+    private static long getHighestCheckpoint(List<TestEvent> events) {
+        return events.stream()
+                .filter(e -> e instanceof CheckpointCompletedEvent)
+                .mapToLong(e -> ((CheckpointCompletedEvent) e).checkpointID)
+                .max()
+                .getAsLong();
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/TestJobExecutor.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/TestJobExecutor.java
new file mode 100644
index 0000000..c9a28e7
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/TestJobExecutor.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommand;
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher;
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher.TestCommandScope;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestCommandAckEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+import org.apache.flink.runtime.testutils.CommonTestUtils;
+import org.apache.flink.test.util.MiniClusterWithClientResource;
+import org.apache.flink.util.SerializedThrowable;
+
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.flink.api.common.JobStatus.FINISHED;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommand.FAIL;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher.TestCommandScope.ALL_SUBTASKS;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher.TestCommandScope.SINGLE_SUBTASK;
+import static org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue.TestEventHandler.TestEventNextAction.CONTINUE;
+import static org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue.TestEventHandler.TestEventNextAction.STOP;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.waitForAllTaskRunning;
+import static org.junit.Assert.fail;
+
+/**
+ * A helper class to control {@link TestJobWithDescription} execution using {@link
+ * TestCommandDispatcher} and {@link TestEventQueue}.
+ */
+class TestJobExecutor {
+    private static final Logger LOG = LoggerFactory.getLogger(TestJobExecutor.class);
+    private final MiniClusterWithClientResource miniClusterResource;
+    private final TestJobWithDescription testJob;
+    private final JobID jobID;
+
+    private TestJobExecutor(
+            TestJobWithDescription testJob,
+            JobID jobID,
+            MiniClusterWithClientResource miniClusterResource) {
+        this.testJob = testJob;
+        this.jobID = jobID;
+        this.miniClusterResource = miniClusterResource;
+    }
+
+    public static TestJobExecutor execute(
+            TestJobWithDescription testJob, MiniClusterWithClientResource miniClusterResource)
+            throws Exception {
+        LOG.debug("submitGraph: {}", testJob.jobGraph);
+        JobID job = miniClusterResource.getClusterClient().submitJob(testJob.jobGraph).get();
+        waitForAllTaskRunning(miniClusterResource.getMiniCluster(), job, false);
+        return new TestJobExecutor(testJob, job, miniClusterResource);
+    }
+
+    public TestJobExecutor waitForAllRunning() throws Exception {
+        LOG.debug("waitForAllRunning in {}", jobID);
+        waitForAllTaskRunning(miniClusterResource.getMiniCluster(), jobID, true);
+        return this;
+    }
+
+    public TestJobExecutor waitForEvent(Class<? extends TestEvent> eventClass) throws Exception {
+        LOG.debug("waitForEvent: {}", eventClass.getSimpleName());
+        testJob.eventQueue.withHandler(
+                e -> eventClass.isAssignableFrom(e.getClass()) ? STOP : CONTINUE);
+        return this;
+    }
+
+    public TestJobExecutor stopWithSavepoint(TemporaryFolder folder, boolean withDrain)
+            throws Exception {
+        LOG.debug("stopWithSavepoint: {} (withDrain: {})", folder, withDrain);
+        ClusterClient<?> client = miniClusterResource.getClusterClient();
+        client.stopWithSavepoint(jobID, withDrain, folder.newFolder().toString()).get();
+        return this;
+    }
+
+    public TestJobExecutor sendOperatorCommand(
+            String operatorID, TestCommand command, TestCommandScope scope) {
+        LOG.debug("send command: {} to {}/{}", command, operatorID, scope);
+        testJob.commandQueue.dispatch(command, scope, operatorID);
+        return this;
+    }
+
+    public void triggerFailover() throws Exception {
+        LOG.debug("sendCommand: {}", FAIL);
+        BlockingQueue<TestEvent> queue = new LinkedBlockingQueue<>();
+        Consumer<TestEvent> listener = queue::add;
+        testJob.eventQueue.addListener(listener);
+        testJob.commandQueue.broadcast(FAIL, SINGLE_SUBTASK);
+        try {
+            waitForFailover(queue);
+        } catch (TimeoutException e) {
+            handleFailoverTimeout(e);
+        } finally {
+            testJob.eventQueue.removeListener(listener);
+        }
+
+        waitForAllRunning();
+    }
+
+    private void waitForFailover(BlockingQueue<TestEvent> queue) throws Exception {
+        int timeoutMs = 10_000;
+        Deadline deadline = Deadline.fromNow(Duration.ofMillis(timeoutMs));
+
+        String operatorId = null;
+        int subtaskId = -1;
+        int attemptNumber = -1;
+
+        while (deadline.hasTimeLeft()) {
+            TestEvent e = queue.poll(deadline.timeLeft().toMillis(), MILLISECONDS);
+            if (e instanceof TestCommandAckEvent) {
+                TestCommandAckEvent ack = (TestCommandAckEvent) e;
+                if (ack.getCommand() == FAIL) {
+                    operatorId = ack.operatorId;
+                    subtaskId = ack.subtaskIndex;
+                    attemptNumber = ack.getAttemptNumber();
+                }
+            } else if (e instanceof OperatorStartedEvent && operatorId != null) {
+                OperatorStartedEvent started = (OperatorStartedEvent) e;
+                if (started.operatorId.equals(operatorId)
+                        && started.subtaskIndex == subtaskId
+                        && started.getAttemptNumber() >= attemptNumber) {
+                    return;
+                }
+            }
+        }
+        throw new TimeoutException("No subtask restarted in " + timeoutMs + "ms");
+    }
+
+    private void handleFailoverTimeout(TimeoutException e) throws Exception {
+        String message =
+                String.format(
+                        "Unable to failover the job: %s; job status: %s",
+                        e.getMessage(),
+                        miniClusterResource.getClusterClient().getJobStatus(jobID).get());
+        Optional<SerializedThrowable> throwable =
+                miniClusterResource
+                        .getClusterClient()
+                        .requestJobResult(jobID)
+                        .get()
+                        .getSerializedThrowable();
+        if (throwable.isPresent()) {
+            throw new RuntimeException(message, throwable.get());
+        } else {
+            throw new RuntimeException(message);
+        }
+    }
+
+    public TestJobExecutor sendBroadcastCommand(TestCommand command, TestCommandScope scope) {
+        LOG.debug("sendCommand: {}", command);
+        testJob.commandQueue.broadcast(command, scope);
+        return this;
+    }
+
+    public TestJobExecutor waitForTermination() throws Exception {
+        LOG.debug("waitForTermination");
+        while (!miniClusterResource
+                .getClusterClient()
+                .getJobStatus(jobID)
+                .get()
+                .isGloballyTerminalState()) {
+            Thread.sleep(1);
+        }
+        return this;
+    }
+
+    public TestJobExecutor assertFinishedSuccessfully() throws Exception {
+        LOG.debug("assertFinishedSuccessfully");
+        JobStatus jobStatus = miniClusterResource.getClusterClient().getJobStatus(jobID).get();
+        if (!jobStatus.equals(FINISHED)) {
+            String message = String.format("Job didn't finish successfully, status: %s", jobStatus);
+            Optional<SerializedThrowable> throwable =
+                    miniClusterResource
+                            .getClusterClient()
+                            .requestJobResult(jobID)
+                            .get()
+                            .getSerializedThrowable();
+            if (throwable.isPresent()) {
+                throw new AssertionError(message, throwable.get());
+            } else {
+                fail(message);
+            }
+        }
+        return this;
+    }
+
+    public TestJobExecutor waitForSubtasksToFinish(JobVertexID id, TestCommandScope scope)
+            throws Exception {
+        LOG.debug("waitForSubtasksToFinish vertex {}, all subtasks: {}", id, scope);
+        CommonTestUtils.waitForSubtasksToFinish(
+                miniClusterResource.getMiniCluster(), jobID, id, scope == ALL_SUBTASKS);
+        return this;
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/TestJobWithDescription.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/TestJobWithDescription.java
new file mode 100644
index 0000000..7f39c65
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/TestJobWithDescription.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle;
+
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.unmodifiableMap;
+import static java.util.Collections.unmodifiableSet;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/** Test {@link JobGraph} with description to allow control its execution and validation. */
+public class TestJobWithDescription {
+    public final JobGraph jobGraph;
+    public final Set<String> sources;
+    public final Set<String> operatorsWithLifecycleTracking;
+    public final Set<String> operatorsWithDataFlowTracking;
+    public final Map<String, Integer> operatorsNumberOfInputs;
+    public final TestEventQueue eventQueue;
+    public final TestCommandDispatcher commandQueue;
+
+    public TestJobWithDescription(
+            JobGraph jobGraph,
+            Set<String> sources,
+            Set<String> operatorsWithLifecycleTracking,
+            Set<String> operatorsWithDataFlowTracking,
+            Map<String, Integer> operatorsNumberOfInputs,
+            TestEventQueue eventQueue,
+            TestCommandDispatcher commandQueue) {
+        this.jobGraph = jobGraph;
+        this.sources = unmodifiableSet(sources);
+        this.operatorsWithLifecycleTracking = unmodifiableSet(operatorsWithLifecycleTracking);
+        this.operatorsWithDataFlowTracking = unmodifiableSet(operatorsWithDataFlowTracking);
+        this.operatorsNumberOfInputs = unmodifiableMap(operatorsNumberOfInputs);
+        this.eventQueue = checkNotNull(eventQueue);
+        this.commandQueue = checkNotNull(commandQueue);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/SharedTestCommandDispatcher.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/SharedTestCommandDispatcher.java
new file mode 100644
index 0000000..3ad5251
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/SharedTestCommandDispatcher.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.command;
+
+import org.apache.flink.testutils.junit.SharedObjects;
+import org.apache.flink.testutils.junit.SharedReference;
+
+/**
+ * {@link TestCommandDispatcher} implementation that delegates to another one wrapped into a {@link
+ * SharedReference} to avoid serialization issues.
+ */
+class SharedTestCommandDispatcher implements TestCommandDispatcher {
+    private final SharedReference<TestCommandDispatcher> ref;
+
+    public SharedTestCommandDispatcher(SharedObjects shared, TestCommandDispatcher delegate) {
+        this.ref = shared.add(delegate);
+    }
+
+    @Override
+    public void dispatch(TestCommand testCommand, TestCommandScope scope, String operatorID) {
+        ref.get().dispatch(testCommand, scope, operatorID);
+    }
+
+    @Override
+    public void subscribe(CommandExecutor executor, String operatorID) {
+        ref.get().subscribe(executor, operatorID);
+    }
+
+    @Override
+    public void broadcast(TestCommand testCommand, TestCommandScope scope) {
+        ref.get().broadcast(testCommand, scope);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommand.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommand.java
new file mode 100644
index 0000000..79d100d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommand.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.command;
+
+/**
+ * A command to be executed by a test operator. Upon execution, it must be acked by sending an
+ * {@link org.apache.flink.runtime.operators.lifecycle.event.TestCommandAckEvent Ack} event.
+ */
+public interface TestCommand {
+    TestCommand FAIL =
+            new TestCommand() {
+                @Override
+                public boolean isTerminal() {
+                    return true;
+                }
+
+                @Override
+                public String toString() {
+                    return "FAIL";
+                }
+            };
+    TestCommand FINISH_SOURCES =
+            new TestCommand() {
+                @Override
+                public boolean isTerminal() {
+                    return true;
+                }
+
+                @Override
+                public String toString() {
+                    return "FINISH_SOURCES";
+                }
+            };
+
+    boolean isTerminal();
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommandDispatcher.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommandDispatcher.java
new file mode 100644
index 0000000..cdf4649
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommandDispatcher.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.command;
+
+import org.apache.flink.testutils.junit.SharedObjects;
+
+import java.io.Serializable;
+
+/**
+ * A dispatcher for {@link TestCommand} executed by operators in a test job. The purpose of sending
+ * commands is decoupling operator logic from control (i.e. test case logic).
+ */
+public interface TestCommandDispatcher extends Serializable {
+
+    void subscribe(CommandExecutor executor, String operatorID);
+
+    void dispatch(TestCommand command, TestCommandScope scope, String operatorID);
+
+    void broadcast(TestCommand command, TestCommandScope scope);
+
+    /** An executor of {@link TestCommand}s. */
+    interface CommandExecutor {
+        void execute(TestCommand testCommand);
+    }
+
+    /** Specifies whether to which subtasks to apply the command. */
+    enum TestCommandScope {
+        SINGLE_SUBTASK,
+        ALL_SUBTASKS
+    }
+
+    static TestCommandDispatcher createShared(SharedObjects shared) {
+        return new SharedTestCommandDispatcher(shared, new TestCommandDispatcherImpl());
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommandDispatcherImpl.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommandDispatcherImpl.java
new file mode 100644
index 0000000..a02483d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/command/TestCommandDispatcherImpl.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.command;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyList;
+
+class TestCommandDispatcherImpl implements TestCommandDispatcher {
+    private final Map<String, List<CommandExecutor>> subscribers = new ConcurrentHashMap<>();
+
+    @Override
+    public void subscribe(CommandExecutor executor, String operatorID) {
+        subscribers.computeIfAbsent(operatorID, ign -> new CopyOnWriteArrayList<>()).add(executor);
+    }
+
+    @Override
+    public void dispatch(TestCommand testCommand, TestCommandScope scope, String operatorID) {
+        executeInternal(testCommand, scope, subscribers.getOrDefault(operatorID, emptyList()));
+    }
+
+    @Override
+    public void broadcast(TestCommand testCommand, TestCommandScope scope) {
+        executeInternal(
+                testCommand,
+                scope,
+                subscribers.values().stream()
+                        .flatMap(Collection::stream)
+                        .collect(Collectors.toList()));
+    }
+
+    private void executeInternal(
+            TestCommand command, TestCommandScope scope, List<CommandExecutor> executors) {
+        Set<CommandExecutor> toRemove = new HashSet<>();
+        for (CommandExecutor executor : executors) {
+            executor.execute(command);
+            if (command.isTerminal()) {
+                toRemove.add(executor);
+            }
+            if (scope == TestCommandScope.SINGLE_SUBTASK) {
+                break;
+            }
+        }
+        executors.removeAll(toRemove);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/CheckpointCompletedEvent.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/CheckpointCompletedEvent.java
new file mode 100644
index 0000000..35718bb
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/CheckpointCompletedEvent.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.state.CheckpointListener;
+
+import java.util.Objects;
+
+/**
+ * An event of calling {@link CheckpointListener#notifyCheckpointComplete(long)
+ * notifyCheckpointComplete} method on an operator.
+ */
+@Internal
+public class CheckpointCompletedEvent extends TestEvent {
+    public final long checkpointID;
+
+    public CheckpointCompletedEvent(String operatorId, int subtaskIndex, long checkpointID) {
+        super(operatorId, subtaskIndex);
+        this.checkpointID = checkpointID;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof CheckpointCompletedEvent)) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        CheckpointCompletedEvent that = (CheckpointCompletedEvent) o;
+        return checkpointID == that.checkpointID;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), checkpointID);
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + "(" + checkpointID + ")";
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/CheckpointStartedEvent.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/CheckpointStartedEvent.java
new file mode 100644
index 0000000..8d8b146
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/CheckpointStartedEvent.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+
+import java.util.Objects;
+
+/**
+ * An event of calling {@link AbstractStreamOperator#snapshotState(StateSnapshotContext)
+ * snapshotState} method on an operator.
+ */
+@Internal
+public class CheckpointStartedEvent extends TestEvent {
+    public final long checkpointID;
+
+    public CheckpointStartedEvent(String operatorId, int subtaskIndex, long checkpointID) {
+        super(operatorId, subtaskIndex);
+        this.checkpointID = checkpointID;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof CheckpointStartedEvent)) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        CheckpointStartedEvent that = (CheckpointStartedEvent) o;
+        return checkpointID == that.checkpointID;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), checkpointID);
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + "(" + checkpointID + ")";
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/InputEndedEvent.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/InputEndedEvent.java
new file mode 100644
index 0000000..47a8115
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/InputEndedEvent.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.operators.BoundedMultiInput;
+
+import java.util.Objects;
+
+/**
+ * An event of calling {@link BoundedMultiInput#endInput(int) endInput} method or the like on an
+ * operator.
+ */
+@Internal
+public class InputEndedEvent extends TestEvent {
+    public final int inputId;
+
+    public InputEndedEvent(String operatorId, int subtaskIndex, int inputId) {
+        super(operatorId, subtaskIndex);
+        this.inputId = inputId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof InputEndedEvent)) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        InputEndedEvent that = (InputEndedEvent) o;
+        return inputId == that.inputId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), inputId);
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + "(" + inputId + ")";
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/OperatorFinishedEvent.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/OperatorFinishedEvent.java
new file mode 100644
index 0000000..093ebb7
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/OperatorFinishedEvent.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.streaming.api.operators.StreamOperator;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An event of calling {@link StreamOperator#finish() finish}. Serves two purposes:
+ *
+ * <ol>
+ *   <li>Verify that the call actually happened
+ *   <li>Collect sent/receive info to verify that there was no data loss
+ * </ol>
+ */
+public class OperatorFinishedEvent extends TestEvent {
+    final LastReceivedVertexDataInfo receiveInfo;
+    public final long lastSent;
+
+    public OperatorFinishedEvent(
+            String operatorId,
+            int subtaskIndex,
+            long lastSent,
+            LastReceivedVertexDataInfo receiveInfo) {
+        super(operatorId, subtaskIndex);
+        this.receiveInfo = receiveInfo;
+        this.lastSent = lastSent;
+    }
+
+    public long getLastReceived(String upstreamID, int upstreamIndex) {
+        return receiveInfo.forUpstream(upstreamID).getOrDefault(upstreamIndex, -1L);
+    }
+
+    /** LastVertexDataInfo. */
+    public static class LastVertexDataInfo implements Serializable {
+        public final Map<Integer, Long> bySubtask = new HashMap<>();
+    }
+
+    /** LastReceivedVertexDataInfo. */
+    public static class LastReceivedVertexDataInfo implements Serializable {
+        private final Map<String, LastVertexDataInfo> byUpstreamOperatorID;
+
+        public LastReceivedVertexDataInfo(Map<String, LastVertexDataInfo> byUpstreamOperatorID) {
+            this.byUpstreamOperatorID = byUpstreamOperatorID;
+        }
+
+        public Map<Integer, Long> forUpstream(String upstreamID) {
+            return byUpstreamOperatorID.getOrDefault(upstreamID, new LastVertexDataInfo())
+                    .bySubtask;
+        }
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/OperatorStartedEvent.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/OperatorStartedEvent.java
new file mode 100644
index 0000000..10dcb7a
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/OperatorStartedEvent.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+/** An event indicating that an operator subtask was started. */
+public class OperatorStartedEvent extends TestEvent {
+    private final int attemptNumber;
+
+    public OperatorStartedEvent(String operatorId, int subtaskIndex, int attemptNumber) {
+        super(operatorId, subtaskIndex);
+        this.attemptNumber = attemptNumber;
+    }
+
+    public int getAttemptNumber() {
+        return attemptNumber;
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + ", attemptNumber=" + attemptNumber;
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/SharedTestEventQueue.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/SharedTestEventQueue.java
new file mode 100644
index 0000000..139e9e3
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/SharedTestEventQueue.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.testutils.junit.SharedReference;
+
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * A {@link TestEventQueue} that delegates to another {@link TestEventQueue} which is shared as
+ * {@link SharedReference} to avoid serialization issues.
+ */
+class SharedTestEventQueue implements TestEventQueue {
+    private final SharedReference<TestEventQueue> delegate;
+
+    public SharedTestEventQueue(SharedReference<TestEventQueue> delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public void add(TestEvent e) {
+        delegate.get().add(e);
+    }
+
+    @Override
+    public List<TestEvent> getAll() {
+        return delegate.get().getAll();
+    }
+
+    @Override
+    public void withHandler(TestEventHandler handler) throws Exception {
+        delegate.get().withHandler(handler);
+    }
+
+    @Override
+    public void addListener(Consumer<TestEvent> listener) {
+        delegate.get().addListener(listener);
+    }
+
+    @Override
+    public void removeListener(Consumer<TestEvent> listener) {
+        delegate.get().removeListener(listener);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestCommandAckEvent.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestCommandAckEvent.java
new file mode 100644
index 0000000..049ff81
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestCommandAckEvent.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommand;
+
+/** An event indicating that a {@link TestCommand} was received and either executed or dropped. */
+public class TestCommandAckEvent extends TestEvent {
+    private final TestCommand command;
+    private final int attemptNumber;
+
+    public TestCommandAckEvent(
+            String operatorId, int subtaskIndex, int attemptNumber, TestCommand command) {
+        super(operatorId, subtaskIndex);
+        this.command = command;
+        this.attemptNumber = attemptNumber;
+    }
+
+    public TestCommand getCommand() {
+        return command;
+    }
+
+    public int getAttemptNumber() {
+        return attemptNumber;
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + ", attemptNumber=" + attemptNumber + ", command=" + command;
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEvent.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEvent.java
new file mode 100644
index 0000000..78e7da5
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEvent.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.annotation.Internal;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * An event of calling some {@link org.apache.flink.streaming.api.operators.StreamOperator} method
+ * or some other event in the operator lifecycle.
+ */
+@Internal
+public abstract class TestEvent implements Serializable {
+    public final String operatorId;
+    public final int subtaskIndex;
+    private static final long serialVersionUID = 1L;
+
+    TestEvent(String operatorId, int subtaskIndex) {
+        this.operatorId = operatorId;
+        this.subtaskIndex = subtaskIndex;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof TestEvent)) {
+            return false;
+        }
+        TestEvent testEvent = (TestEvent) o;
+        return subtaskIndex == testEvent.subtaskIndex && operatorId.equals(testEvent.operatorId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(operatorId, subtaskIndex);
+    }
+
+    @Override
+    public String toString() {
+        return String.format(
+                "%s %s/%d",
+                getClass().getSimpleName().replaceAll("Event\\b", ""), operatorId, subtaskIndex);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEventQueue.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEventQueue.java
new file mode 100644
index 0000000..0370f02
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEventQueue.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.testutils.junit.SharedObjects;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * A queue for {@link TestEvent}. The purpose is to allow to react to operator lifecycle events
+ * externally. Events are not collected in the job output (e.g. accumulators) because they can be
+ * emitted when the outputs are closed.
+ */
+public interface TestEventQueue extends Serializable {
+    void add(TestEvent e);
+
+    List<TestEvent> getAll();
+
+    void withHandler(TestEventHandler handler) throws Exception;
+
+    void addListener(Consumer<TestEvent> listener);
+
+    void removeListener(Consumer<TestEvent> listener);
+
+    /** Handler of {@link TestEvent}s. */
+    interface TestEventHandler {
+        enum TestEventNextAction {
+            CONTINUE,
+            STOP
+        }
+
+        TestEventNextAction handle(TestEvent e);
+    }
+
+    static TestEventQueue createShared(SharedObjects sharedObjects) {
+        return new SharedTestEventQueue(sharedObjects.add(new TestEventQueueImpl()));
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEventQueueImpl.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEventQueueImpl.java
new file mode 100644
index 0000000..3608e68
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/TestEventQueueImpl.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue.TestEventHandler.TestEventNextAction;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.function.Consumer;
+
+import static org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue.TestEventHandler.TestEventNextAction.CONTINUE;
+
+class TestEventQueueImpl implements TestEventQueue {
+    private final List<TestEvent> events = new CopyOnWriteArrayList<>();
+    private final List<Consumer<TestEvent>> listeners = new CopyOnWriteArrayList<>();
+
+    public void add(TestEvent e) {
+        events.add(e);
+        listeners.forEach(l -> l.accept(e));
+    }
+
+    @Override
+    public void withHandler(TestEventHandler handler) throws Exception {
+        BlockingQueue<TestEvent> queue = new LinkedBlockingQueue<>();
+        Consumer<TestEvent> listener = queue::add;
+        addListener(listener);
+        try {
+            for (TestEventNextAction nextAction = CONTINUE; nextAction == CONTINUE; ) {
+                nextAction = handler.handle(queue.take());
+            }
+        } finally {
+            removeListener(listener);
+        }
+    }
+
+    public void removeListener(Consumer<TestEvent> listener) {
+        listeners.remove(listener);
+    }
+
+    public void addListener(Consumer<TestEvent> listener) {
+        listeners.add(listener);
+    }
+
+    @Override
+    public List<TestEvent> getAll() {
+        return Collections.unmodifiableList(events);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/WatermarkReceivedEvent.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/WatermarkReceivedEvent.java
new file mode 100644
index 0000000..5d2613f
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/event/WatermarkReceivedEvent.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.event;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+
+import java.util.Objects;
+
+/**
+ * An event of calling {@link AbstractStreamOperator#processWatermark(Watermark) processWatermark}
+ * or its variants.
+ */
+@Internal
+public class WatermarkReceivedEvent extends TestEvent {
+    public final long ts;
+    public final int inputId;
+
+    public WatermarkReceivedEvent(String operatorId, int subtaskIndex, long ts, int inputId) {
+        super(operatorId, subtaskIndex);
+        this.ts = ts;
+        this.inputId = inputId;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof WatermarkReceivedEvent)) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        WatermarkReceivedEvent that = (WatermarkReceivedEvent) o;
+        return ts == that.ts && inputId == that.inputId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), ts, inputId);
+    }
+
+    @Override
+    public String toString() {
+        return String.format("%s(%d/%d)", super.toString(), ts, inputId);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/MultiInputTestOperator.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/MultiInputTestOperator.java
new file mode 100644
index 0000000..110ed30
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/MultiInputTestOperator.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.graph;
+
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommand;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.InputEndedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent.LastReceivedVertexDataInfo;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+import org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperatorV2;
+import org.apache.flink.streaming.api.operators.BoundedMultiInput;
+import org.apache.flink.streaming.api.operators.Input;
+import org.apache.flink.streaming.api.operators.MultipleInputStreamOperator;
+import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
+import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * {@link MultipleInputStreamOperator} that emits {@link TestEvent}s and reacts to {@link
+ * TestCommand}s. It processes the data by passing it downstream replacing the relevant fields.
+ */
+@SuppressWarnings("rawtypes") // Input type parameter not allowed by interface
+class MultiInputTestOperator extends AbstractStreamOperatorV2<TestDataElement>
+        implements MultipleInputStreamOperator<TestDataElement>,
+                ProcessingTimeCallback,
+                BoundedMultiInput {
+    private final String operatorId;
+    private final List<Input> inputs;
+    private final TestEventQueue eventQueue;
+    // atomic only to update from different static classes, not for // concurrency
+    private final AtomicLong lastDataSent;
+    private final Map<String, OperatorFinishedEvent.LastVertexDataInfo> lastDataReceived;
+
+    public MultiInputTestOperator(
+            int numInputs,
+            StreamOperatorParameters<TestDataElement> params,
+            TestEventQueue eventQueue,
+            String operatorId) {
+        super(params, numInputs);
+        this.lastDataReceived = new HashMap<>();
+        this.lastDataSent = new AtomicLong(0L);
+        this.inputs =
+                IntStream.rangeClosed(1, numInputs)
+                        .mapToObj(
+                                id ->
+                                        new TestEventInput(
+                                                id,
+                                                eventQueue,
+                                                output,
+                                                operatorId,
+                                                getRuntimeContext().getIndexOfThisSubtask(),
+                                                lastDataReceived,
+                                                lastDataSent))
+                        .collect(Collectors.toList());
+        this.eventQueue = eventQueue;
+        this.operatorId = operatorId;
+    }
+
+    @Override
+    public void open() throws Exception {
+        super.open();
+        registerTimer();
+        this.eventQueue.add(
+                new OperatorStartedEvent(
+                        operatorId,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        getRuntimeContext().getAttemptNumber()));
+    }
+
+    @Override
+    public List<Input> getInputs() {
+        return inputs;
+    }
+
+    @Override
+    public void finish() throws Exception {
+        eventQueue.add(
+                new OperatorFinishedEvent(
+                        operatorId,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        lastDataSent.get(),
+                        new LastReceivedVertexDataInfo(lastDataReceived)));
+        super.finish();
+    }
+
+    @Override
+    public void snapshotState(StateSnapshotContext context) throws Exception {
+        eventQueue.add(
+                new CheckpointStartedEvent(
+                        operatorId,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        context.getCheckpointId()));
+        super.snapshotState(context);
+    }
+
+    @Override
+    public void notifyCheckpointComplete(long checkpointId) throws Exception {
+        eventQueue.add(
+                new CheckpointCompletedEvent(
+                        operatorId, getRuntimeContext().getIndexOfThisSubtask(), checkpointId));
+        super.notifyCheckpointComplete(checkpointId);
+    }
+
+    @Override
+    public void onProcessingTime(long timestamp) {
+        registerTimer();
+    }
+
+    private void registerTimer() {
+        getProcessingTimeService()
+                .registerTimer(getProcessingTimeService().getCurrentProcessingTime() + 1, this);
+    }
+
+    @Override
+    public void endInput(int inputId) throws Exception {
+        eventQueue.add(
+                new InputEndedEvent(
+                        operatorId, getRuntimeContext().getIndexOfThisSubtask(), inputId));
+    }
+
+    private static class TestEventInput implements Input<TestDataElement> {
+        private final int id;
+        private final TestEventQueue eventQueue;
+        private final Output<StreamRecord<TestDataElement>> output;
+        private final String operatorId;
+        private final int subtaskIndex;
+        private final Map<String, OperatorFinishedEvent.LastVertexDataInfo> lastDataReceived;
+        private final AtomicLong lastDataSent;
+
+        public TestEventInput(
+                int id,
+                TestEventQueue eventQueue,
+                Output<StreamRecord<TestDataElement>> output,
+                String operatorId,
+                int subtaskIndex,
+                Map<String, OperatorFinishedEvent.LastVertexDataInfo> lastDataReceived,
+                AtomicLong lastDataSent) {
+            this.id = id;
+            this.eventQueue = eventQueue;
+            this.output = output;
+            this.operatorId = operatorId;
+            this.subtaskIndex = subtaskIndex;
+            this.lastDataReceived = lastDataReceived;
+            this.lastDataSent = lastDataSent;
+        }
+
+        @Override
+        public void processElement(StreamRecord<TestDataElement> element) throws Exception {
+            TestDataElement e = element.getValue();
+            lastDataReceived
+                    .computeIfAbsent(
+                            e.operatorId, ign -> new OperatorFinishedEvent.LastVertexDataInfo())
+                    .bySubtask
+                    .put(e.subtaskIndex, e.seq);
+            output.collect(
+                    new StreamRecord<>(
+                            new TestDataElement(
+                                    operatorId, subtaskIndex, lastDataSent.incrementAndGet())));
+        }
+
+        @Override
+        public void processWatermark(Watermark mark) throws Exception {
+            eventQueue.add(
+                    new WatermarkReceivedEvent(operatorId, subtaskIndex, mark.getTimestamp(), id));
+            output.emitWatermark(mark);
+        }
+
+        @Override
+        public void processWatermarkStatus(WatermarkStatus watermarkStatus) {}
+
+        @Override
+        public void processLatencyMarker(LatencyMarker latencyMarker) {}
+
+        @Override
+        public void setKeyContextElement(StreamRecord<TestDataElement> record) {}
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/MultiInputTestOperatorFactory.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/MultiInputTestOperatorFactory.java
new file mode 100644
index 0000000..393dde2
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/MultiInputTestOperatorFactory.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.graph;
+
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.operators.StreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
+
+class MultiInputTestOperatorFactory implements StreamOperatorFactory<TestDataElement> {
+    private final int numInputs;
+    private final TestEventQueue eventQueue;
+    private ChainingStrategy strategy;
+    private final String operatorId;
+
+    public MultiInputTestOperatorFactory(
+            int numInputs, TestEventQueue eventQueue, String operatorId) {
+        this.numInputs = numInputs;
+        this.eventQueue = eventQueue;
+        this.operatorId = operatorId;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public <T extends StreamOperator<TestDataElement>> T createStreamOperator(
+            StreamOperatorParameters<TestDataElement> parameters) {
+        return (T) new MultiInputTestOperator(numInputs, parameters, eventQueue, operatorId);
+    }
+
+    @Override
+    public void setChainingStrategy(ChainingStrategy strategy) {
+        this.strategy = strategy;
+    }
+
+    @Override
+    public ChainingStrategy getChainingStrategy() {
+        return strategy;
+    }
+
+    @Override
+    public Class<? extends StreamOperator<?>> getStreamOperatorClass(ClassLoader classLoader) {
+        return MultiInputTestOperator.class;
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/OneInputTestStreamOperator.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/OneInputTestStreamOperator.java
new file mode 100644
index 0000000..96d6239
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/OneInputTestStreamOperator.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.graph;
+
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommand;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.InputEndedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent.LastVertexDataInfo;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+import org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.BoundedOneInput;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * {@link OneInputStreamOperator} that emits {@link TestEvent}s and reacts to {@link TestCommand}s.
+ * It processes the data by passing it downstream replacing the relevant fields.
+ */
+class OneInputTestStreamOperator extends AbstractStreamOperator<TestDataElement>
+        implements OneInputStreamOperator<TestDataElement, TestDataElement>,
+                BoundedOneInput,
+                ProcessingTimeCallback {
+    private final String operatorID;
+    private long lastDataSent;
+    private final Map<String, LastVertexDataInfo> lastDataReceived = new HashMap<>();
+    private boolean timerRegistered;
+    private final TestEventQueue eventQueue;
+
+    OneInputTestStreamOperator(String operatorID, TestEventQueue eventQueue) {
+        this.operatorID = operatorID;
+        this.eventQueue = eventQueue;
+    }
+
+    @Override
+    public void open() throws Exception {
+        super.open();
+        this.eventQueue.add(
+                new OperatorStartedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        getRuntimeContext().getAttemptNumber()));
+    }
+
+    @Override
+    public void snapshotState(StateSnapshotContext context) throws Exception {
+        eventQueue.add(
+                new CheckpointStartedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        context.getCheckpointId()));
+        super.snapshotState(context);
+    }
+
+    @Override
+    public void notifyCheckpointComplete(long checkpointId) throws Exception {
+        eventQueue.add(
+                new CheckpointCompletedEvent(
+                        operatorID, getRuntimeContext().getIndexOfThisSubtask(), checkpointId));
+        super.notifyCheckpointComplete(checkpointId);
+    }
+
+    @Override
+    public void finish() throws Exception {
+        eventQueue.add(
+                new OperatorFinishedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        lastDataSent,
+                        new OperatorFinishedEvent.LastReceivedVertexDataInfo(lastDataReceived)));
+        super.finish();
+    }
+
+    @Override
+    public void processElement(StreamRecord<TestDataElement> element) throws Exception {
+        TestDataElement e = element.getValue();
+        lastDataReceived
+                .computeIfAbsent(e.operatorId, ign -> new LastVertexDataInfo())
+                .bySubtask
+                .put(e.subtaskIndex, e.seq);
+        output.collect(
+                new StreamRecord<>(
+                        new TestDataElement(
+                                operatorID,
+                                getRuntimeContext().getIndexOfThisSubtask(),
+                                ++lastDataSent)));
+        if (!timerRegistered) {
+            registerTimer();
+            timerRegistered = true;
+        }
+    }
+
+    @Override
+    public void processWatermark(Watermark mark) throws Exception {
+        eventQueue.add(
+                new WatermarkReceivedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        mark.getTimestamp(),
+                        1));
+        super.processWatermark(mark);
+    }
+
+    @Override
+    public void endInput() throws Exception {
+        eventQueue.add(
+                new InputEndedEvent(operatorID, getRuntimeContext().getIndexOfThisSubtask(), 1));
+    }
+
+    @Override
+    public void onProcessingTime(long timestamp) {
+        registerTimer();
+    }
+
+    private void registerTimer() {
+        getProcessingTimeService()
+                .registerTimer(getProcessingTimeService().getCurrentProcessingTime() + 1, this);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/OneInputTestStreamOperatorFactory.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/OneInputTestStreamOperatorFactory.java
new file mode 100644
index 0000000..f3a0243
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/OneInputTestStreamOperatorFactory.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.graph;
+
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeServiceAware;
+
+class OneInputTestStreamOperatorFactory
+        implements OneInputStreamOperatorFactory<TestDataElement, TestDataElement>,
+                ProcessingTimeServiceAware {
+    private ChainingStrategy strategy = ChainingStrategy.DEFAULT_CHAINING_STRATEGY;
+    private ProcessingTimeService processingTimeService;
+    private final String operatorID;
+    private final TestEventQueue eventQueue;
+
+    OneInputTestStreamOperatorFactory(String operatorID, TestEventQueue eventQueue) {
+        this.operatorID = operatorID;
+        this.eventQueue = eventQueue;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public <T extends StreamOperator<TestDataElement>> T createStreamOperator(
+            StreamOperatorParameters<TestDataElement> parameters) {
+        OneInputTestStreamOperator operator =
+                new OneInputTestStreamOperator(operatorID, eventQueue);
+        operator.setup(
+                parameters.getContainingTask(),
+                parameters.getStreamConfig(),
+                parameters.getOutput());
+        operator.setProcessingTimeService(processingTimeService);
+        return (T) operator;
+    }
+
+    @Override
+    public void setChainingStrategy(ChainingStrategy strategy) {
+        this.strategy = strategy;
+    }
+
+    @Override
+    public ChainingStrategy getChainingStrategy() {
+        return strategy;
+    }
+
+    @Override
+    public Class<? extends StreamOperator> getStreamOperatorClass(ClassLoader classLoader) {
+        return OneInputTestStreamOperator.class;
+    }
+
+    @Override
+    public void setProcessingTimeService(ProcessingTimeService processingTimeService) {
+        this.processingTimeService = processingTimeService;
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestDataElement.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestDataElement.java
new file mode 100644
index 0000000..75aeb6d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestDataElement.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.graph;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.operators.Input;
+
+import java.util.Objects;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * An event of sending some data element downstream, usually from {@link Input#processElement
+ * processElement} method on an operator.
+ */
+@Internal
+public class TestDataElement {
+    public final String operatorId;
+    public final int subtaskIndex;
+    public final long seq;
+
+    public TestDataElement(String operatorId, int subtaskIndex, long seq) {
+        this.operatorId = checkNotNull(operatorId);
+        this.subtaskIndex = subtaskIndex;
+        this.seq = seq;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof TestDataElement)) {
+            return false;
+        }
+        TestDataElement that = (TestDataElement) o;
+        return subtaskIndex == that.subtaskIndex
+                && seq == that.seq
+                && operatorId.equals(that.operatorId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(operatorId, subtaskIndex, seq);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestEventSource.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestEventSource.java
new file mode 100644
index 0000000..c899e80
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestEventSource.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.graph;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommand;
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent.LastReceivedVertexDataInfo;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestCommandAckEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
+import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+
+import java.util.Queue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommand.FAIL;
+import static org.apache.flink.runtime.operators.lifecycle.command.TestCommand.FINISH_SOURCES;
+
+/**
+ * {@link SourceFunction} that emits {@link TestEvent}s and reacts to {@link TestCommand}s. It emits
+ * {@link TestDataElement} to its output.
+ */
+class TestEventSource extends RichSourceFunction<TestDataElement>
+        implements ParallelSourceFunction<TestDataElement> {
+    private final String operatorID;
+    private final TestCommandDispatcher commandQueue;
+    private transient Queue<TestCommand> scheduledCommands;
+    private transient volatile boolean isRunning = true;
+    private final TestEventQueue eventQueue;
+
+    TestEventSource(
+            String operatorID, TestEventQueue eventQueue, TestCommandDispatcher commandQueue) {
+        this.operatorID = operatorID;
+        this.eventQueue = eventQueue;
+        this.commandQueue = commandQueue;
+    }
+
+    @Override
+    public void open(Configuration parameters) throws Exception {
+        super.open(parameters);
+        this.isRunning = true;
+        this.scheduledCommands = new LinkedBlockingQueue<>();
+        this.commandQueue.subscribe(cmd -> scheduledCommands.add(cmd), operatorID);
+        this.eventQueue.add(
+                new OperatorStartedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        getRuntimeContext().getAttemptNumber()));
+    }
+
+    @Override
+    public void run(SourceContext<TestDataElement> ctx) {
+        long lastSent = 0;
+        while (isRunning) {
+            TestCommand cmd = scheduledCommands.poll();
+            if (cmd == FINISH_SOURCES) {
+                ack(cmd);
+                isRunning = false;
+            } else if (cmd == FAIL) {
+                ack(cmd);
+                throw new RuntimeException("requested to fail");
+            } else if (cmd == null) {
+                synchronized (ctx.getCheckpointLock()) {
+                    ctx.collect(
+                            new TestDataElement(
+                                    operatorID,
+                                    getRuntimeContext().getIndexOfThisSubtask(),
+                                    ++lastSent));
+                }
+            } else {
+                throw new RuntimeException("unknown command " + cmd);
+            }
+        }
+        // note: this only gets collected with FLIP-147 changes
+        synchronized (ctx.getCheckpointLock()) {
+            eventQueue.add(
+                    new OperatorFinishedEvent(
+                            operatorID,
+                            getRuntimeContext().getIndexOfThisSubtask(),
+                            lastSent,
+                            new LastReceivedVertexDataInfo(emptyMap())));
+        }
+    }
+
+    private void ack(TestCommand cmd) {
+        eventQueue.add(
+                new TestCommandAckEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        getRuntimeContext().getAttemptNumber(),
+                        cmd));
+    }
+
+    @Override
+    public void cancel() {
+        isRunning = false;
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestJobBuilders.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestJobBuilders.java
new file mode 100644
index 0000000..eb2ab1a
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TestJobBuilders.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.graph;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription;
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommandDispatcher;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.MultipleConnectedStreams;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.streaming.api.transformations.MultipleInputTransformation;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.testutils.junit.SharedObjects;
+import org.apache.flink.util.function.TriFunction;
+
+import javax.annotation.Nullable;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static org.apache.flink.api.common.restartstrategy.RestartStrategies.noRestart;
+import static org.apache.flink.configuration.JobManagerOptions.EXECUTION_FAILOVER_STRATEGY;
+
+/** Helper to build {@link TestJobWithDescription}. */
+public class TestJobBuilders {
+
+    /** {@link TestJobWithDescription} builder. */
+    public interface TestingGraphBuilder
+            extends TriFunction<
+                    SharedObjects,
+                    Consumer<Configuration>,
+                    Consumer<StreamExecutionEnvironment>,
+                    TestJobWithDescription> {}
+
+    private TestJobBuilders() {}
+
+    public static final TestingGraphBuilder SIMPLE_GRAPH_BUILDER =
+            new TestingGraphBuilder() {
+                @Override
+                public TestJobWithDescription apply(
+                        SharedObjects shared,
+                        Consumer<Configuration> confConsumer,
+                        Consumer<StreamExecutionEnvironment> envConsumer) {
+
+                    TestEventQueue eventQueue = TestEventQueue.createShared(shared);
+                    TestCommandDispatcher commandQueue = TestCommandDispatcher.createShared(shared);
+                    StreamExecutionEnvironment env = prepareEnv(confConsumer, envConsumer);
+
+                    // using hashes so that operators emit identifiable events
+                    String unitedSourceLeft = OP_ID_HASH_PREFIX + "1";
+                    String mapForward = OP_ID_HASH_PREFIX + "5";
+
+                    DataStream<TestDataElement> src =
+                            env.addSource(
+                                            new TestEventSource(
+                                                    unitedSourceLeft, eventQueue, commandQueue))
+                                    .setUidHash(unitedSourceLeft)
+                                    .assignTimestampsAndWatermarks(createWmAssigner());
+
+                    SingleOutputStreamOperator<TestDataElement> forwardTransform =
+                            src.transform(
+                                            "transform-1-forward",
+                                            TypeInformation.of(TestDataElement.class),
+                                            new OneInputTestStreamOperatorFactory(
+                                                    mapForward, eventQueue))
+                                    .setUidHash(mapForward);
+
+                    forwardTransform.addSink(new DiscardingSink<>());
+
+                    Map<String, Integer> operatorsNumberOfInputs = new HashMap<>();
+                    operatorsNumberOfInputs.put(mapForward, 1);
+
+                    return new TestJobWithDescription(
+                            env.getStreamGraph().getJobGraph(),
+                            singleton(unitedSourceLeft),
+                            new HashSet<>(singletonList(mapForward)),
+                            new HashSet<>(asList(unitedSourceLeft, mapForward)),
+                            operatorsNumberOfInputs,
+                            eventQueue,
+                            commandQueue);
+                }
+
+                @Override
+                public String toString() {
+                    return "simple graph";
+                }
+            };
+
+    public static final TestingGraphBuilder COMPLEX_GRAPH_BUILDER =
+            new TestingGraphBuilder() {
+                @Override
+                public TestJobWithDescription apply(
+                        SharedObjects shared,
+                        Consumer<Configuration> confConsumer,
+                        Consumer<StreamExecutionEnvironment> envConsumer) {
+
+                    TestEventQueue eventQueue = TestEventQueue.createShared(shared);
+                    TestCommandDispatcher commandQueue = TestCommandDispatcher.createShared(shared);
+
+                    StreamExecutionEnvironment env = prepareEnv(confConsumer, envConsumer);
+
+                    // using hashes so that operators emit identifiable events
+                    String unitedSourceLeft = OP_ID_HASH_PREFIX + "1";
+                    String unitedSourceRight = OP_ID_HASH_PREFIX + "2";
+                    String connectedSource = OP_ID_HASH_PREFIX + "3";
+                    String multiSource = OP_ID_HASH_PREFIX + "4";
+                    String mapForward = OP_ID_HASH_PREFIX + "5";
+                    String mapKeyed = OP_ID_HASH_PREFIX + "6";
+                    String mapTwoInput = OP_ID_HASH_PREFIX + "7";
+                    String multipleInput = OP_ID_HASH_PREFIX + "8";
+
+                    // todo: FLIP-27 sources
+                    // todo: chain sources
+                    DataStream<TestDataElement> unitedSources =
+                            env.addSource(
+                                            new TestEventSource(
+                                                    unitedSourceLeft, eventQueue, commandQueue))
+                                    .setUidHash(unitedSourceLeft)
+                                    .assignTimestampsAndWatermarks(createWmAssigner())
+                                    .union(
+                                            env.addSource(
+                                                            new TestEventSource(
+                                                                    unitedSourceRight,
+                                                                    eventQueue,
+                                                                    commandQueue))
+                                                    .setUidHash(unitedSourceRight)
+                                                    .assignTimestampsAndWatermarks(
+                                                            createWmAssigner()));
+                    SingleOutputStreamOperator<TestDataElement> sideSource =
+                            env.addSource(
+                                            new TestEventSource(
+                                                    multiSource, eventQueue, commandQueue))
+                                    .setUidHash(multiSource)
+                                    .assignTimestampsAndWatermarks(createWmAssigner());
+
+                    DataStream<?>[] inputs = new DataStream[] {unitedSources, sideSource};
+                    final MultipleInputTransformation<TestDataElement> multipleInputsTransform =
+                            new MultipleInputTransformation<>(
+                                    "MultipleInputOperator",
+                                    new MultiInputTestOperatorFactory(
+                                            inputs.length, eventQueue, multipleInput),
+                                    TypeInformation.of(TestDataElement.class),
+                                    env.getParallelism());
+                    for (DataStream<?> input : inputs) {
+                        multipleInputsTransform.addInput(input.getTransformation());
+                    }
+                    multipleInputsTransform.setChainingStrategy(ChainingStrategy.HEAD_WITH_SOURCES);
+                    env.addOperator(multipleInputsTransform);
+
+                    SingleOutputStreamOperator<TestDataElement> multipleSources =
+                            new MultipleConnectedStreams(env)
+                                    .transform(multipleInputsTransform)
+                                    .setUidHash(multiSource);
+
+                    SingleOutputStreamOperator<TestDataElement> forwardTransform =
+                            multipleSources
+                                    .startNewChain()
+                                    .transform(
+                                            "transform-1-forward",
+                                            TypeInformation.of(TestDataElement.class),
+                                            new OneInputTestStreamOperatorFactory(
+                                                    mapForward, eventQueue))
+                                    .setUidHash(mapForward);
+
+                    SingleOutputStreamOperator<TestDataElement> keyedTransform =
+                            forwardTransform
+                                    .startNewChain()
+                                    .keyBy(e -> e)
+                                    .transform(
+                                            "transform-2-keyed",
+                                            TypeInformation.of(TestDataElement.class),
+                                            new OneInputTestStreamOperatorFactory(
+                                                    mapKeyed, eventQueue))
+                                    .setUidHash(mapKeyed);
+
+                    SingleOutputStreamOperator<TestDataElement> twoInputTransform =
+                            keyedTransform
+                                    .startNewChain()
+                                    .connect(
+                                            env.addSource(
+                                                            new TestEventSource(
+                                                                    connectedSource,
+                                                                    eventQueue,
+                                                                    commandQueue))
+                                                    .setUidHash(connectedSource))
+                                    .transform(
+                                            "transform-3-two-input",
+                                            TypeInformation.of(TestDataElement.class),
+                                            new TwoInputTestStreamOperator(mapTwoInput, eventQueue))
+                                    .setUidHash(mapTwoInput);
+
+                    twoInputTransform.addSink(new DiscardingSink<>());
+
+                    Map<String, Integer> operatorsNumberOfInputs = new HashMap<>();
+                    operatorsNumberOfInputs.put(mapForward, 1);
+                    operatorsNumberOfInputs.put(mapKeyed, 1);
+                    operatorsNumberOfInputs.put(mapTwoInput, 2);
+                    operatorsNumberOfInputs.put(multipleInput, 2);
+
+                    return new TestJobWithDescription(
+                            env.getStreamGraph().getJobGraph(),
+                            new HashSet<>(
+                                    asList(unitedSourceLeft, unitedSourceRight, connectedSource)),
+                            new HashSet<>(asList(mapForward, mapKeyed, mapTwoInput, multipleInput)),
+                            new HashSet<>(
+                                    asList(
+                                            unitedSourceLeft,
+                                            unitedSourceRight,
+                                            connectedSource,
+                                            mapForward,
+                                            mapKeyed,
+                                            mapTwoInput,
+                                            multipleInput)),
+                            operatorsNumberOfInputs,
+                            eventQueue,
+                            commandQueue);
+                }
+
+                @Override
+                public String toString() {
+                    return "complex graph";
+                }
+            };
+
+    private static StreamExecutionEnvironment prepareEnv(
+            Consumer<Configuration> confConsumer,
+            Consumer<StreamExecutionEnvironment> envConsumer) {
+        Configuration configuration = new Configuration();
+        configuration.set(EXECUTION_FAILOVER_STRATEGY, "full");
+        confConsumer.accept(configuration);
+        StreamExecutionEnvironment env =
+                StreamExecutionEnvironment.getExecutionEnvironment(configuration);
+        env.setParallelism(4);
+        env.setRestartStrategy(noRestart());
+        env.enableCheckpointing(200); // shouldn't matter
+        env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
+        env.getConfig().setAutoWatermarkInterval(50);
+        envConsumer.accept(env);
+        return env;
+    }
+
+    private static final String OP_ID_HASH_PREFIX = "0000000000000000000000000000000";
+
+    @SuppressWarnings("deprecation")
+    private static AssignerWithPeriodicWatermarks<TestDataElement> createWmAssigner() {
+        return new AssignerWithPeriodicWatermarks<TestDataElement>() {
+            private Watermark watermark;
+
+            @Nullable
+            @Override
+            public Watermark getCurrentWatermark() {
+                return watermark;
+            }
+
+            @Override
+            public long extractTimestamp(TestDataElement element, long recordTimestamp) {
+                if (element instanceof TestDataElement) {
+                    this.watermark = new Watermark(((TestDataElement) element).seq);
+                    return ((TestDataElement) element).seq;
+                }
+                return recordTimestamp;
+            }
+        };
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TwoInputTestStreamOperator.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TwoInputTestStreamOperator.java
new file mode 100644
index 0000000..3d8a289
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/graph/TwoInputTestStreamOperator.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.graph;
+
+import org.apache.flink.runtime.operators.lifecycle.command.TestCommand;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.InputEndedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent.LastReceivedVertexDataInfo;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEventQueue;
+import org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent;
+import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.BoundedMultiInput;
+import org.apache.flink.streaming.api.operators.TwoInputStreamOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * {@link TwoInputStreamOperator} that emits {@link TestEvent}s and reacts to {@link TestCommand}s.
+ * It processes the data by passing it downstream replacing the relevant fields.
+ */
+class TwoInputTestStreamOperator extends AbstractStreamOperator<TestDataElement>
+        implements TwoInputStreamOperator<TestDataElement, TestDataElement, TestDataElement>,
+                BoundedMultiInput,
+                ProcessingTimeCallback {
+    private final String operatorID;
+    private long lastDataSent;
+    private final Map<String, OperatorFinishedEvent.LastVertexDataInfo> lastDataReceived =
+            new HashMap<>();
+    private boolean timerRegistered;
+    private final TestEventQueue eventQueue;
+
+    TwoInputTestStreamOperator(String operatorID, TestEventQueue eventQueue) {
+        this.operatorID = operatorID;
+        this.eventQueue = eventQueue;
+    }
+
+    @Override
+    public void open() throws Exception {
+        super.open();
+        this.eventQueue.add(
+                new OperatorStartedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        getRuntimeContext().getAttemptNumber()));
+    }
+
+    @Override
+    public void snapshotState(StateSnapshotContext context) throws Exception {
+        collectEvent(
+                new CheckpointStartedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        context.getCheckpointId()));
+        super.snapshotState(context);
+    }
+
+    @Override
+    public void notifyCheckpointComplete(long checkpointId) throws Exception {
+        collectEvent(
+                new CheckpointCompletedEvent(
+                        operatorID, getRuntimeContext().getIndexOfThisSubtask(), checkpointId));
+        super.notifyCheckpointComplete(checkpointId);
+    }
+
+    @Override
+    public void finish() throws Exception {
+        collectEvent(
+                new OperatorFinishedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        lastDataSent,
+                        new LastReceivedVertexDataInfo(lastDataReceived)));
+        super.finish();
+    }
+
+    private void processElement(StreamRecord<TestDataElement> element) {
+        TestDataElement e = element.getValue();
+        lastDataReceived
+                .computeIfAbsent(
+                        e.operatorId, ign -> new OperatorFinishedEvent.LastVertexDataInfo())
+                .bySubtask
+                .put(e.subtaskIndex, e.seq);
+        output.collect(
+                new StreamRecord<>(
+                        new TestDataElement(
+                                operatorID,
+                                getRuntimeContext().getIndexOfThisSubtask(),
+                                ++lastDataSent)));
+        if (!timerRegistered) {
+            registerTimer();
+            timerRegistered = true;
+        }
+    }
+
+    @Override
+    public void processWatermark1(Watermark mark) throws Exception {
+        collectEvent(
+                new WatermarkReceivedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        mark.getTimestamp(),
+                        1));
+        super.processWatermark1(mark);
+    }
+
+    @Override
+    public void processWatermark2(Watermark mark) throws Exception {
+        collectEvent(
+                new WatermarkReceivedEvent(
+                        operatorID,
+                        getRuntimeContext().getIndexOfThisSubtask(),
+                        mark.getTimestamp(),
+                        2));
+        super.processWatermark2(mark);
+    }
+
+    @Override
+    public void onProcessingTime(long timestamp) {
+        registerTimer();
+    }
+
+    private void registerTimer() {
+        getProcessingTimeService()
+                .registerTimer(getProcessingTimeService().getCurrentProcessingTime() + 1, this);
+    }
+
+    @Override
+    public void endInput(int inputId) throws Exception {
+        collectEvent(
+                new InputEndedEvent(
+                        operatorID, getRuntimeContext().getIndexOfThisSubtask(), inputId));
+    }
+
+    @Override
+    public void processElement1(StreamRecord element) {
+        processElement(element);
+    }
+
+    @Override
+    public void processElement2(StreamRecord element) {
+        processElement(element);
+    }
+
+    private void collectEvent(TestEvent e) {
+        eventQueue.add(e);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/DrainingValidator.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/DrainingValidator.java
new file mode 100644
index 0000000..c6e3f59
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/DrainingValidator.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.validation;
+
+import org.apache.flink.runtime.OperatorIDPair;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription;
+import org.apache.flink.runtime.operators.lifecycle.event.InputEndedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent;
+import org.apache.flink.streaming.api.watermark.Watermark;
+
+import java.util.BitSet;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * For each input, checks that the {@link Watermark#MAX_WATERMARK} was received and then the input
+ * was closed.
+ */
+public class DrainingValidator implements TestOperatorLifecycleValidator {
+
+    @Override
+    public void validateOperatorLifecycle(
+            TestJobWithDescription job,
+            String operatorId,
+            int subtaskIndex,
+            List<TestEvent> operatorEvents) {
+        BitSet endedInputs = new BitSet();
+        BitSet inputsWithMaxWatermark = new BitSet();
+        for (TestEvent ev : operatorEvents) {
+            if (ev instanceof WatermarkReceivedEvent) {
+                WatermarkReceivedEvent w = (WatermarkReceivedEvent) ev;
+                if (w.ts == Watermark.MAX_WATERMARK.getTimestamp()) {
+                    assertFalse(inputsWithMaxWatermark.get(w.inputId));
+                    inputsWithMaxWatermark.set(w.inputId);
+                }
+            } else if (ev instanceof InputEndedEvent) {
+                InputEndedEvent w = (InputEndedEvent) ev;
+                assertTrue(
+                        format(
+                                "Input %d ended before receiving max watermark by %s[%d]",
+                                w.inputId, operatorId, subtaskIndex),
+                        inputsWithMaxWatermark.get(w.inputId));
+                assertFalse(endedInputs.get(w.inputId));
+                endedInputs.set(w.inputId);
+            }
+        }
+        assertEquals(
+                format("Incorrect number of ended inputs for %s[%d]", operatorId, subtaskIndex),
+                getNumInputs(job, operatorId),
+                endedInputs.cardinality());
+    }
+
+    private static int getNumInputs(TestJobWithDescription testJob, String operator) {
+        Integer explicitNumInputs = testJob.operatorsNumberOfInputs.get(operator);
+        if (explicitNumInputs != null) {
+            return explicitNumInputs;
+        }
+        Iterable<JobVertex> vertices = testJob.jobGraph.getVertices();
+        for (JobVertex vertex : vertices) {
+            for (OperatorIDPair p : vertex.getOperatorIDs()) {
+                OperatorID operatorID =
+                        p.getUserDefinedOperatorID().orElse(p.getGeneratedOperatorID());
+                if (operatorID.toString().equals(operator)) {
+                    // warn: this returns the number of network inputs
+                    // which may not coincide with logical
+                    // e.g. single-input operator after two sources united
+                    // will have two network inputs
+                    return vertex.getNumberOfInputs();
+                }
+            }
+        }
+        throw new NoSuchElementException(operator);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/FinishingValidator.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/FinishingValidator.java
new file mode 100644
index 0000000..d5289a9
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/FinishingValidator.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.validation;
+
+import org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.WatermarkReceivedEvent;
+import org.apache.flink.streaming.api.operators.StreamOperator;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static java.lang.String.format;
+import static java.util.stream.Collectors.toList;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Checks that {@link StreamOperator#finish()} was called and then only checkpoints were starting or
+ * completing and at least one of them completed.
+ */
+public class FinishingValidator implements TestOperatorLifecycleValidator {
+
+    @Override
+    public void validateOperatorLifecycle(
+            TestJobWithDescription job,
+            String operatorId,
+            int subtaskIndex,
+            List<TestEvent> operatorEvents) {
+        boolean opFinished = false;
+        Set<Long> finalCheckpointCandidates = new HashSet<>();
+        for (TestEvent ev : operatorEvents) {
+            if (ev instanceof OperatorFinishedEvent) {
+                opFinished = true;
+            } else if (ev instanceof CheckpointStartedEvent) {
+                if (opFinished) {
+                    finalCheckpointCandidates.add(((CheckpointStartedEvent) ev).checkpointID);
+                }
+            } else if (ev instanceof CheckpointCompletedEvent) {
+                if (finalCheckpointCandidates.contains(
+                        ((CheckpointCompletedEvent) ev).checkpointID)) {
+                    return;
+                }
+            } else if (opFinished) {
+                fail(
+                        format(
+                                "Unexpected event after operator %s[%d] finished: %s",
+                                operatorId, subtaskIndex, ev));
+            }
+        }
+        assertTrue(
+                format(
+                        "Operator %s[%d] wasn't finished (events: %s)",
+                        operatorId, subtaskIndex, operatorEvents),
+                opFinished);
+        fail(
+                format(
+                        "Operator %s[%d] was finished but didn't finish the checkpoint after that;"
+                                + "checkpoints started after finish: %s (events (excluding watermarks): %s)",
+                        operatorId,
+                        subtaskIndex,
+                        finalCheckpointCandidates,
+                        operatorEvents.stream()
+                                .filter(ev -> !(ev instanceof WatermarkReceivedEvent))
+                                .collect(toList())));
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/SameCheckpointValidator.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/SameCheckpointValidator.java
new file mode 100644
index 0000000..9a8b68a
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/SameCheckpointValidator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.validation;
+
+import org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointCompletedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.CheckpointStartedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+
+import java.util.List;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/** Checks that all subtasks have received the same last checkpoint. */
+public class SameCheckpointValidator implements TestOperatorLifecycleValidator {
+
+    private final long lastCheckpointID;
+
+    public SameCheckpointValidator(long lastCheckpointID) {
+        this.lastCheckpointID = lastCheckpointID;
+    }
+
+    @Override
+    public void validateOperatorLifecycle(
+            TestJobWithDescription job,
+            String operatorId,
+            int subtaskIndex,
+            List<TestEvent> operatorEvents) {
+
+        boolean started = false;
+        boolean finished = false;
+        for (TestEvent ev : operatorEvents) {
+            if (ev instanceof CheckpointStartedEvent) {
+                if (lastCheckpointID == ((CheckpointStartedEvent) ev).checkpointID) {
+                    assertFalse(
+                            format(
+                                    "Operator %s[%d] started checkpoint %d twice",
+                                    operatorId, subtaskIndex, lastCheckpointID),
+                            started);
+                    started = true;
+                }
+            } else if (ev instanceof CheckpointCompletedEvent) {
+                if (lastCheckpointID == ((CheckpointCompletedEvent) ev).checkpointID) {
+                    assertTrue(
+                            format(
+                                    "Operator %s[%d] finished checkpoint %d before starting",
+                                    operatorId, subtaskIndex, lastCheckpointID),
+                            started);
+                    assertFalse(
+                            format(
+                                    "Operator %s[%d] finished checkpoint %d twice",
+                                    operatorId, subtaskIndex, lastCheckpointID),
+                            finished);
+                    finished = true;
+                }
+            }
+        }
+        assertTrue(
+                format(
+                        "Operator %s[%d] didn't finish checkpoint %d (events: %s)",
+                        operatorId, subtaskIndex, lastCheckpointID, operatorEvents),
+                finished);
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/TestJobDataFlowValidator.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/TestJobDataFlowValidator.java
new file mode 100644
index 0000000..7a8f789
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/TestJobDataFlowValidator.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.validation;
+
+import org.apache.flink.runtime.OperatorIDPair;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSet;
+import org.apache.flink.runtime.jobgraph.JobEdge;
+import org.apache.flink.runtime.jobgraph.JobVertex;
+import org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription;
+import org.apache.flink.runtime.operators.lifecycle.event.OperatorFinishedEvent;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Optional;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/** Check that all data from the upstream reached the respective downstreams. */
+public class TestJobDataFlowValidator {
+    private static final Logger LOG = LoggerFactory.getLogger(TestJobDataFlowValidator.class);
+
+    public static void checkDataFlow(TestJobWithDescription testJob) {
+        Map<String, Map<Integer, OperatorFinishedEvent>> finishEvents = new HashMap<>();
+        for (TestEvent ev : testJob.eventQueue.getAll()) {
+            if (ev instanceof OperatorFinishedEvent) {
+                finishEvents
+                        .computeIfAbsent(ev.operatorId, ign -> new HashMap<>())
+                        .put(ev.subtaskIndex, ((OperatorFinishedEvent) ev));
+            }
+        }
+
+        for (JobVertex upstream : testJob.jobGraph.getVertices()) {
+            for (IntermediateDataSet produced : upstream.getProducedDataSets()) {
+                for (JobEdge edge : produced.getConsumers()) {
+                    Optional<String> upstreamID = getTrackedOperatorID(upstream, true, testJob);
+                    Optional<String> downstreamID =
+                            getTrackedOperatorID(edge.getTarget(), false, testJob);
+                    if (upstreamID.isPresent() && downstreamID.isPresent()) {
+                        checkDataFlow(upstreamID.get(), downstreamID.get(), edge, finishEvents);
+                    } else {
+                        LOG.debug("Ignoring edge (untracked operator): {}", edge);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Check that for each upstream subtask there exists a downstream subtask that received it's
+     * latest emitted element.
+     */
+    private static void checkDataFlow(
+            String upstreamID,
+            String downstreamID,
+            JobEdge edge,
+            Map<String, Map<Integer, OperatorFinishedEvent>> finishEvents) {
+        LOG.debug(
+                "Checking {} edge\n  from {} ({})\n  to {} ({})",
+                edge.getDistributionPattern(),
+                edge.getSource().getProducer().getName(),
+                upstreamID,
+                edge.getTarget().getName(),
+                downstreamID);
+
+        Map<Integer, OperatorFinishedEvent> downstreamFinishInfo =
+                getForOperator(downstreamID, finishEvents);
+
+        Map<Integer, OperatorFinishedEvent> upstreamFinishInfo =
+                getForOperator(upstreamID, finishEvents);
+
+        upstreamFinishInfo.forEach(
+                (upstreamIndex, upstreamInfo) ->
+                        assertTrue(
+                                String.format(
+                                        "No downstream received %s from %s[%d]; received: %s",
+                                        upstreamInfo.lastSent,
+                                        upstreamID,
+                                        upstreamIndex,
+                                        downstreamFinishInfo),
+                                anySubtaskReceived(
+                                        upstreamID,
+                                        upstreamIndex,
+                                        upstreamInfo.lastSent,
+                                        downstreamFinishInfo.values())));
+    }
+
+    private static boolean anySubtaskReceived(
+            String upstreamID,
+            int upstreamIndex,
+            long upstreamValue,
+            Collection<OperatorFinishedEvent> downstreamFinishInfo) {
+        return downstreamFinishInfo.stream()
+                .anyMatch(
+                        event -> event.getLastReceived(upstreamID, upstreamIndex) == upstreamValue);
+    }
+
+    private static Map<Integer, OperatorFinishedEvent> getForOperator(
+            String operatorId, Map<String, Map<Integer, OperatorFinishedEvent>> finishEvents) {
+        Map<Integer, OperatorFinishedEvent> events = finishEvents.get(operatorId);
+        assertNotNull(
+                format(
+                        "Operator finish info wasn't collected: %s (collected: %s)",
+                        operatorId, finishEvents),
+                events);
+        return events;
+    }
+
+    /**
+     * Traverse operators in the chain in the vertex and return the first tracked operator ID. For
+     * upstream, start with head, for downstream - with tail (see {@link
+     * JobVertex#getOperatorIDs()}). If a chain doesn't contain any tracked operators return
+     * nothing.
+     */
+    private static Optional<String> getTrackedOperatorID(
+            JobVertex vertex, boolean upstream, TestJobWithDescription testJob) {
+        ListIterator<OperatorIDPair> iterator =
+                vertex.getOperatorIDs().listIterator(upstream ? 0 : vertex.getOperatorIDs().size());
+        while (upstream ? iterator.hasNext() : iterator.hasPrevious()) {
+            OperatorIDPair idPair = upstream ? iterator.next() : iterator.previous();
+            String id =
+                    idPair.getUserDefinedOperatorID()
+                            .orElse(idPair.getGeneratedOperatorID())
+                            .toString();
+            if (testJob.operatorsWithDataFlowTracking.contains(id)) {
+                return Optional.of(id);
+            }
+        }
+        return Optional.empty();
+    }
+}
diff --git a/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/TestOperatorLifecycleValidator.java b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/TestOperatorLifecycleValidator.java
new file mode 100644
index 0000000..9b68fb0
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/runtime/operators/lifecycle/validation/TestOperatorLifecycleValidator.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.operators.lifecycle.validation;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.runtime.operators.lifecycle.TestJobWithDescription;
+import org.apache.flink.runtime.operators.lifecycle.event.TestEvent;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Validates operator lifecycle based on collected {@link TestEvent}s. */
+public interface TestOperatorLifecycleValidator {
+
+    void validateOperatorLifecycle(
+            TestJobWithDescription job,
+            String operatorId,
+            int subtaskIndex,
+            List<TestEvent> operatorEvents);
+
+    static void checkOperatorsLifecycle(
+            TestJobWithDescription testJob, TestOperatorLifecycleValidator... validators) {
+        Map<Tuple2<String, Integer>, List<TestEvent>> eventsByOperator = new HashMap<>();
+        for (TestEvent ev : testJob.eventQueue.getAll()) {
+            eventsByOperator
+                    .computeIfAbsent(
+                            Tuple2.of(ev.operatorId, ev.subtaskIndex), ign -> new ArrayList<>())
+                    .add(ev);
+        }
+        eventsByOperator.forEach(
+                (operatorIdAndIndex, operatorEvents) -> {
+                    String id = operatorIdAndIndex.f0;
+                    if (testJob.operatorsWithLifecycleTracking.contains(id)) {
+                        for (TestOperatorLifecycleValidator validator : validators) {
+                            validator.validateOperatorLifecycle(
+                                    testJob, id, operatorIdAndIndex.f1, operatorEvents);
+                        }
+                    }
+                });
+    }
+}