You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2021/10/15 08:09:16 UTC

[flink] 03/04: [FLINK-23704][task] FLIP-27 sources are not generating LatencyMarkers

This is an automated email from the ASF dual-hosted git repository.

pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 223d6888836dc9007bf55e3000f531b126f0d0fb
Author: SteNicholas <pr...@163.com>
AuthorDate: Wed Sep 22 17:10:49 2021 +0800

    [FLINK-23704][task] FLIP-27 sources are not generating LatencyMarkers
---
 .../streaming/api/operators/SourceOperator.java    | 102 +++++++
 .../source/SourceOperatorEventTimeTest.java        |  21 ++
 .../SourceOperatorLatencyMetricsTest.java          | 332 +++++++++++++++++++++
 .../flink/streaming/util/CollectorDataOutput.java  |  69 +++++
 4 files changed, 524 insertions(+)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
index 72ae6cd..2190b0f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
@@ -29,10 +29,12 @@ import org.apache.flink.api.connector.source.SourceReader;
 import org.apache.flink.api.connector.source.SourceReaderContext;
 import org.apache.flink.api.connector.source.SourceSplit;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
 import org.apache.flink.core.io.InputStatus;
 import org.apache.flink.core.io.SimpleVersionedSerializer;
 import org.apache.flink.metrics.groups.SourceReaderMetricGroup;
 import org.apache.flink.runtime.io.AvailabilityProvider;
+import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.runtime.metrics.groups.InternalSourceReaderMetricGroup;
 import org.apache.flink.runtime.operators.coordination.OperatorEvent;
 import org.apache.flink.runtime.operators.coordination.OperatorEventGateway;
@@ -49,6 +51,7 @@ import org.apache.flink.streaming.api.operators.source.TimestampsAndWatermarks;
 import org.apache.flink.streaming.api.operators.util.SimpleVersionedListState;
 import org.apache.flink.streaming.runtime.io.DataInputStatus;
 import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput;
+import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
 import org.apache.flink.streaming.runtime.tasks.StreamTask;
@@ -57,11 +60,15 @@ import org.apache.flink.util.FlinkRuntimeException;
 import org.apache.flink.util.UserCodeClassLoader;
 import org.apache.flink.util.function.FunctionWithException;
 
+import javax.annotation.Nullable;
+
 import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledFuture;
 
 import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
 
 /**
  * Base source operator only used for integrating the source reader which is proposed by FLIP-27. It
@@ -149,6 +156,8 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
 
     private InternalSourceReaderMetricGroup sourceMetricGroup;
 
+    private LatencyMarkerEmitter<OUT> latencyMarerEmitter;
+
     public SourceOperator(
             FunctionWithException<SourceReaderContext, SourceReader<OUT, SplitT>, Exception>
                     readerFactory,
@@ -282,6 +291,19 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
                             watermarkStrategy, sourceMetricGroup);
         }
 
+        latencyMarerEmitter =
+                new LatencyMarkerEmitter<>(
+                        getProcessingTimeService(),
+                        getExecutionConfig().isLatencyTrackingConfigured()
+                                ? getExecutionConfig().getLatencyTrackingInterval()
+                                : getContainingTask()
+                                        .getEnvironment()
+                                        .getTaskManagerInfo()
+                                        .getConfiguration()
+                                        .getLong(MetricOptions.LATENCY_INTERVAL),
+                        getOperatorID(),
+                        getRuntimeContext().getIndexOfThisSubtask());
+
         // restore the state if necessary.
         final List<SplitT> splits = CollectionUtil.iterableToList(readerState.get());
         if (!splits.isEmpty()) {
@@ -296,6 +318,7 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
         sourceReader.start();
 
         eventTimeLogic.startPeriodicWatermarkEmits();
+        latencyMarerEmitter.startLatencyMarkerEmit();
     }
 
     @Override
@@ -303,6 +326,9 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
         if (eventTimeLogic != null) {
             eventTimeLogic.stopPeriodicWatermarkEmits();
         }
+        if (latencyMarerEmitter != null) {
+            latencyMarerEmitter.stopLatencyMarkerEmit();
+        }
         super.finish();
 
         finished.complete(null);
@@ -348,6 +374,7 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
         switch (operatingMode) {
             case OUTPUT_NOT_INITIALIZED:
                 currentMainOutput = eventTimeLogic.createMainOutput(output);
+                latencyMarerEmitter.emitMainOutput(output);
                 lastInvokedOutput = output;
                 this.operatingMode = OperatingMode.READING;
                 return convertToInternalStatus(sourceReader.pollNext(currentMainOutput));
@@ -478,4 +505,79 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
             this.forcedStopFuture.complete(null);
         }
     }
+
+    private static class LatencyMarkerEmitter<OUT> {
+
+        private final ProcessingTimeService timeService;
+
+        private final long latencyTrackingInterval;
+
+        private final OperatorID operatorId;
+
+        private final int subtaskIndex;
+
+        @Nullable private DataOutput<OUT> currentMainOutput;
+
+        @Nullable private ScheduledFuture<?> latencyMarkerTimer;
+
+        public LatencyMarkerEmitter(
+                final ProcessingTimeService timeService,
+                long latencyTrackingInterval,
+                final OperatorID operatorId,
+                final int subtaskIndex) {
+            this.timeService = timeService;
+            this.latencyTrackingInterval = latencyTrackingInterval;
+            this.operatorId = operatorId;
+            this.subtaskIndex = subtaskIndex;
+        }
+
+        // ------------------------------------------------------------------------
+
+        public void emitMainOutput(PushingAsyncDataInput.DataOutput<OUT> output) {
+            // At the moment, we assume only one output is ever created!
+            // This assumption is strict, currently, because many of the classes in this
+            // implementation
+            // do not support re-assigning the underlying output
+            checkState(currentMainOutput == null, "Main output has already been set.");
+            currentMainOutput = output;
+        }
+
+        public void startLatencyMarkerEmit() {
+            checkState(
+                    latencyMarkerTimer == null, "Latency marker emitter has already been started");
+            if (latencyTrackingInterval == 0) {
+                // a value of zero means not activated
+                return;
+            }
+            latencyMarkerTimer =
+                    timeService.scheduleWithFixedDelay(
+                            this::triggerLatencyMarkerEmit, 0L, latencyTrackingInterval);
+        }
+
+        public void stopLatencyMarkerEmit() {
+            if (latencyMarkerTimer != null) {
+                latencyMarkerTimer.cancel(false);
+                latencyMarkerTimer = null;
+            }
+        }
+
+        void triggerLatencyMarkerEmit(@SuppressWarnings("unused") long wallClockTimestamp) {
+            if (currentMainOutput != null) {
+                try {
+                    // ProcessingTimeService callbacks are executed under the
+                    // checkpointing lock
+                    currentMainOutput.emitLatencyMarker(
+                            new LatencyMarker(
+                                    timeService.getCurrentProcessingTime(),
+                                    operatorId,
+                                    subtaskIndex));
+                } catch (Throwable t) {
+                    // we catch the Throwable here so that we don't trigger the
+                    // processing
+                    // timer services async exception handler
+                    LOG.warn("Error while emitting latency marker.", t);
+                }
+            }
+        }
+    }
 }
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/SourceOperatorEventTimeTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/SourceOperatorEventTimeTest.java
index 39f5de4..c0a84f9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/SourceOperatorEventTimeTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/SourceOperatorEventTimeTest.java
@@ -18,22 +18,30 @@
 
 package org.apache.flink.streaming.api.operators.source;
 
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.eventtime.Watermark;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.state.OperatorStateStore;
 import org.apache.flink.api.connector.source.ReaderOutput;
 import org.apache.flink.api.connector.source.SourceReader;
 import org.apache.flink.api.connector.source.mocks.MockSourceSplit;
+import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.fs.CloseableRegistry;
 import org.apache.flink.core.io.InputStatus;
 import org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder;
+import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider;
 import org.apache.flink.runtime.state.StateInitializationContext;
 import org.apache.flink.runtime.state.StateInitializationContextImpl;
+import org.apache.flink.runtime.state.TestTaskStateManager;
 import org.apache.flink.runtime.state.memory.MemoryStateBackend;
 import org.apache.flink.streaming.api.operators.SourceOperator;
 import org.apache.flink.streaming.runtime.io.DataInputStatus;
 import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
+import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask;
+import org.apache.flink.streaming.runtime.tasks.StreamMockEnvironment;
 import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService;
+import org.apache.flink.streaming.util.MockOutput;
+import org.apache.flink.streaming.util.MockStreamConfig;
 
 import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
 
@@ -243,6 +251,19 @@ public class SourceOperatorEventTimeTest {
         final SourceOperator<T, MockSourceSplit> sourceOperator =
                 new TestingSourceOperator<>(
                         reader, watermarkStrategy, timeService, emitProgressiveWatermarks);
+
+        sourceOperator.setup(
+                new SourceOperatorStreamTask<Integer>(
+                        new StreamMockEnvironment(
+                                new Configuration(),
+                                new Configuration(),
+                                new ExecutionConfig(),
+                                1L,
+                                new MockInputSplitProvider(),
+                                1,
+                                new TestTaskStateManager())),
+                new MockStreamConfig(new Configuration(), 1),
+                new MockOutput<>(new ArrayList<>()));
         sourceOperator.initializeState(stateContext);
         sourceOperator.open();
 
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/SourceOperatorLatencyMetricsTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/SourceOperatorLatencyMetricsTest.java
new file mode 100644
index 0000000..7685927
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/SourceOperatorLatencyMetricsTest.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.operators;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.state.OperatorStateStore;
+import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.mocks.MockSourceReader;
+import org.apache.flink.api.connector.source.mocks.MockSourceSplit;
+import org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.io.SimpleVersionedSerialization;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.runtime.operators.coordination.MockOperatorEventGateway;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
+import org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder;
+import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider;
+import org.apache.flink.runtime.state.AbstractStateBackend;
+import org.apache.flink.runtime.state.StateInitializationContext;
+import org.apache.flink.runtime.state.StateInitializationContextImpl;
+import org.apache.flink.runtime.state.TestTaskStateManager;
+import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
+import org.apache.flink.streaming.api.graph.StreamConfig;
+import org.apache.flink.streaming.api.operators.SourceOperator;
+import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
+import org.apache.flink.streaming.runtime.tasks.OperatorChain;
+import org.apache.flink.streaming.runtime.tasks.RegularOperatorChain;
+import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask;
+import org.apache.flink.streaming.runtime.tasks.StreamMockEnvironment;
+import org.apache.flink.streaming.runtime.tasks.StreamTask;
+import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService;
+import org.apache.flink.streaming.util.CollectorDataOutput;
+import org.apache.flink.streaming.util.MockOutput;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/** Tests for the emission of latency markers by {@link SourceOperator} operators. */
+public class SourceOperatorLatencyMetricsTest extends TestLogger {
+
+    private static final long maxProcessingTime = 100L;
+    private static final long latencyMarkInterval = 10L;
+
+    /** Verifies that by default no latency metrics are emitted. */
+    @Test
+    public void testLatencyMarkEmissionDisabled() throws Exception {
+        testLatencyMarkEmission(
+                0,
+                operator ->
+                        setupSourceOperator(
+                                operator,
+                                new ExecutionConfig(),
+                                MockEnvironment.builder().build()));
+    }
+
+    /** Verifies that latency metrics can be enabled via the {@link ExecutionConfig}. */
+    @Test
+    public void testLatencyMarkEmissionEnabledViaExecutionConfig() throws Exception {
+        testLatencyMarkEmission(
+                (int) (maxProcessingTime / latencyMarkInterval) + 1,
+                operator -> {
+                    ExecutionConfig executionConfig = new ExecutionConfig();
+                    executionConfig.setLatencyTrackingInterval(latencyMarkInterval);
+                    setupSourceOperator(
+                            operator, executionConfig, MockEnvironment.builder().build());
+                });
+    }
+
+    /** Verifies that latency metrics can be enabled via the configuration. */
+    @Test
+    public void testLatencyMarkEmissionEnabledViaFlinkConfig() throws Exception {
+        testLatencyMarkEmission(
+                (int) (maxProcessingTime / latencyMarkInterval) + 1,
+                operator -> {
+                    Configuration tmConfig = new Configuration();
+                    tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);
+                    Environment env =
+                            MockEnvironment.builder()
+                                    .setTaskManagerRuntimeInfo(
+                                            new TestingTaskManagerRuntimeInfo(tmConfig))
+                                    .build();
+                    setupSourceOperator(operator, new ExecutionConfig(), env);
+                });
+    }
+
+    /**
+     * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are
+     * disabled via the configuration.
+     */
+    @Test
+    public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
+        testLatencyMarkEmission(
+                (int) (maxProcessingTime / latencyMarkInterval) + 1,
+                operator -> {
+                    ExecutionConfig executionConfig = new ExecutionConfig();
+                    executionConfig.setLatencyTrackingInterval(latencyMarkInterval);
+                    Configuration tmConfig = new Configuration();
+                    tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);
+                    Environment env =
+                            MockEnvironment.builder()
+                                    .setTaskManagerRuntimeInfo(
+                                            new TestingTaskManagerRuntimeInfo(tmConfig))
+                                    .build();
+                    setupSourceOperator(operator, executionConfig, env);
+                });
+    }
+
+    /**
+     * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they
+     * are enabled via the configuration.
+     */
+    @Test
+    public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
+        testLatencyMarkEmission(
+                0,
+                operator -> {
+                    Configuration tmConfig = new Configuration();
+                    tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);
+                    Environment env =
+                            MockEnvironment.builder()
+                                    .setTaskManagerRuntimeInfo(
+                                            new TestingTaskManagerRuntimeInfo(tmConfig))
+                                    .build();
+                    ExecutionConfig executionConfig = new ExecutionConfig();
+                    executionConfig.setLatencyTrackingInterval(0);
+                    setupSourceOperator(operator, executionConfig, env);
+                });
+    }
+
+    private interface OperatorSetupOperation {
+        void setupSourceOperator(SourceOperator<Integer, ?> operator);
+    }
+
+    private void testLatencyMarkEmission(
+            int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
+        final List<StreamElement> output = new ArrayList<>();
+
+        final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
+        testProcessingTimeService.setCurrentTime(0L);
+        final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);
+
+        // regular source operator
+        final SourceOperator<Integer, MockSourceSplit> sourceOperator =
+                new SourceOperator<>(
+                        (context) ->
+                                new ProcessingTimeServiceSourceReader(
+                                        testProcessingTimeService, processingTimes),
+                        new MockOperatorEventGateway(),
+                        new MockSourceSplitSerializer(),
+                        WatermarkStrategy.noWatermarks(),
+                        testProcessingTimeService,
+                        new Configuration(),
+                        "localhost",
+                        true /* emit progressive watermarks */);
+        operatorSetup.setupSourceOperator(sourceOperator);
+
+        // run and wait to be stopped
+        OperatorChain<?, ?> operatorChain =
+                new RegularOperatorChain<>(
+                        sourceOperator.getContainingTask(),
+                        StreamTask.createRecordWriterDelegate(
+                                sourceOperator.getOperatorConfig(),
+                                new MockEnvironmentBuilder().build()));
+        try {
+            sourceOperator.initializeState(getStateContext());
+            sourceOperator.open();
+            sourceOperator.emitNext(new CollectorDataOutput<>(output));
+            sourceOperator.finish();
+        } finally {
+            operatorChain.close();
+        }
+
+        assertEquals(numberLatencyMarkers, output.size());
+
+        long timestamp = 0L;
+        int expectedLatencyIndex = 0;
+
+        int index = 0;
+        // verify that its only latency markers
+        for (; index < numberLatencyMarkers; index++) {
+            StreamElement streamElement = output.get(index);
+            assertTrue(streamElement.isLatencyMarker());
+            assertEquals(
+                    sourceOperator.getOperatorID(),
+                    streamElement.asLatencyMarker().getOperatorId());
+            assertEquals(0, streamElement.asLatencyMarker().getSubtaskIndex());
+
+            // determines the next latency mark that should've been emitted
+            // latency marks are emitted once per latencyMarkInterval,
+            // as a result of which we never emit both 10 and 11
+            while (timestamp > processingTimes.get(expectedLatencyIndex)) {
+                expectedLatencyIndex++;
+            }
+            assertEquals(
+                    processingTimes.get(expectedLatencyIndex).longValue(),
+                    streamElement.asLatencyMarker().getMarkedTime());
+
+            timestamp += latencyMarkInterval;
+        }
+    }
+
+    // ---------------- helper methods -------------------------
+
+    private static <T> void setupSourceOperator(
+            SourceOperator<T, ?> sourceOperator,
+            ExecutionConfig executionConfig,
+            Environment environment) {
+        StreamConfig streamConfig = new StreamConfig(new Configuration());
+        streamConfig.setOperatorID(new OperatorID());
+        try {
+            sourceOperator.setup(
+                    new SourceOperatorStreamTask<Integer>(
+                            getTestingEnvironment(executionConfig, environment)),
+                    streamConfig,
+                    new MockOutput<>(new ArrayList<>()));
+        } catch (Exception e) {
+            e.printStackTrace();
+            fail(e.getMessage());
+        }
+    }
+
+    private StateInitializationContext getStateContext() throws Exception {
+        // Create a mock split.
+        byte[] serializedSplitWithVersion =
+                SimpleVersionedSerialization.writeVersionAndSerialize(
+                        new MockSourceSplitSerializer(), new MockSourceSplit(1234, 10));
+
+        // Crate the state context.
+        OperatorStateStore operatorStateStore = createOperatorStateStore();
+        StateInitializationContext stateContext =
+                new StateInitializationContextImpl(null, operatorStateStore, null, null, null);
+
+        // Update the context.
+        stateContext
+                .getOperatorStateStore()
+                .getListState(
+                        new ListStateDescriptor<>(
+                                "SourceReaderState", BytePrimitiveArraySerializer.INSTANCE))
+                .update(Collections.singletonList(serializedSplitWithVersion));
+
+        return stateContext;
+    }
+
+    private OperatorStateStore createOperatorStateStore() throws Exception {
+        MockEnvironment env = new MockEnvironmentBuilder().build();
+        final AbstractStateBackend abstractStateBackend = new HashMapStateBackend();
+        CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+        return abstractStateBackend.createOperatorStateBackend(
+                env, "test-operator", Collections.emptyList(), cancelStreamRegistry);
+    }
+
+    private static Environment getTestingEnvironment(
+            ExecutionConfig executionConfig, Environment environment) {
+        StreamMockEnvironment mockEnvironment =
+                new StreamMockEnvironment(
+                        new Configuration(),
+                        new Configuration(),
+                        executionConfig,
+                        1L,
+                        new MockInputSplitProvider(),
+                        1,
+                        new TestTaskStateManager());
+        mockEnvironment.setTaskManagerInfo(environment.getTaskManagerInfo());
+        return mockEnvironment;
+    }
+
+    // ------------------------------------------------------------------------
+
+    private static final class ProcessingTimeServiceSourceReader extends MockSourceReader {
+
+        private final TestProcessingTimeService processingTimeService;
+        private final List<Long> processingTimes;
+
+        private boolean closed = false;
+
+        private ProcessingTimeServiceSourceReader(
+                TestProcessingTimeService processingTimeService, List<Long> processingTimes) {
+            this.processingTimeService = processingTimeService;
+            this.processingTimes = processingTimes;
+        }
+
+        @Override
+        public InputStatus pollNext(ReaderOutput<Integer> sourceOutput) throws Exception {
+            for (Long processingTime : processingTimes) {
+                if (closed) {
+                    break;
+                }
+                processingTimeService.setCurrentTime(processingTime);
+            }
+            return super.pollNext(sourceOutput);
+        }
+
+        @Override
+        public void close() throws Exception {
+            closed = true;
+            super.close();
+        }
+    }
+}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorDataOutput.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorDataOutput.java
new file mode 100644
index 0000000..2fb2ee5
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorDataOutput.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput;
+import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
+import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.util.List;
+
+/** Collecting {@link DataOutput} for {@link StreamRecord}. */
+public class CollectorDataOutput<T> implements DataOutput<T> {
+
+    private final List<StreamElement> streamElements;
+
+    public CollectorDataOutput(List<StreamElement> streamElements) {
+        this.streamElements = streamElements;
+    }
+
+    @Override
+    public void emitRecord(StreamRecord<T> streamRecord) throws Exception {
+        try {
+            ClassLoader classLoader = streamRecord.getClass().getClassLoader();
+            T copied =
+                    InstantiationUtil.deserializeObject(
+                            InstantiationUtil.serializeObject(streamRecord.getValue()),
+                            classLoader);
+            streamElements.add(streamRecord.copy(copied));
+        } catch (IOException | ClassNotFoundException ex) {
+            throw new RuntimeException("Unable to deserialize record: " + streamRecord, ex);
+        }
+    }
+
+    @Override
+    public void emitWatermark(Watermark watermark) throws Exception {
+        streamElements.add(watermark);
+    }
+
+    @Override
+    public void emitWatermarkStatus(WatermarkStatus watermarkStatus) throws Exception {
+        streamElements.add(watermarkStatus);
+    }
+
+    @Override
+    public void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception {
+        streamElements.add(latencyMarker);
+    }
+}