You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2021/10/15 08:09:16 UTC
[flink] 03/04: [FLINK-23704][task] FLIP-27 sources are not
generating LatencyMarkers
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 223d6888836dc9007bf55e3000f531b126f0d0fb
Author: SteNicholas <pr...@163.com>
AuthorDate: Wed Sep 22 17:10:49 2021 +0800
[FLINK-23704][task] FLIP-27 sources are not generating LatencyMarkers
---
.../streaming/api/operators/SourceOperator.java | 102 +++++++
.../source/SourceOperatorEventTimeTest.java | 21 ++
.../SourceOperatorLatencyMetricsTest.java | 332 +++++++++++++++++++++
.../flink/streaming/util/CollectorDataOutput.java | 69 +++++
4 files changed, 524 insertions(+)
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
index 72ae6cd..2190b0f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
@@ -29,10 +29,12 @@ import org.apache.flink.api.connector.source.SourceReader;
import org.apache.flink.api.connector.source.SourceReaderContext;
import org.apache.flink.api.connector.source.SourceSplit;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
import org.apache.flink.core.io.InputStatus;
import org.apache.flink.core.io.SimpleVersionedSerializer;
import org.apache.flink.metrics.groups.SourceReaderMetricGroup;
import org.apache.flink.runtime.io.AvailabilityProvider;
+import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.metrics.groups.InternalSourceReaderMetricGroup;
import org.apache.flink.runtime.operators.coordination.OperatorEvent;
import org.apache.flink.runtime.operators.coordination.OperatorEventGateway;
@@ -49,6 +51,7 @@ import org.apache.flink.streaming.api.operators.source.TimestampsAndWatermarks;
import org.apache.flink.streaming.api.operators.util.SimpleVersionedListState;
import org.apache.flink.streaming.runtime.io.DataInputStatus;
import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput;
+import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
import org.apache.flink.streaming.runtime.tasks.StreamTask;
@@ -57,11 +60,15 @@ import org.apache.flink.util.FlinkRuntimeException;
import org.apache.flink.util.UserCodeClassLoader;
import org.apache.flink.util.function.FunctionWithException;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.util.List;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledFuture;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
/**
* Base source operator only used for integrating the source reader which is proposed by FLIP-27. It
@@ -149,6 +156,8 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
private InternalSourceReaderMetricGroup sourceMetricGroup;
+ private LatencyMarkerEmitter<OUT> latencyMarerEmitter;
+
public SourceOperator(
FunctionWithException<SourceReaderContext, SourceReader<OUT, SplitT>, Exception>
readerFactory,
@@ -282,6 +291,19 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
watermarkStrategy, sourceMetricGroup);
}
+ latencyMarerEmitter =
+ new LatencyMarkerEmitter<>(
+ getProcessingTimeService(),
+ getExecutionConfig().isLatencyTrackingConfigured()
+ ? getExecutionConfig().getLatencyTrackingInterval()
+ : getContainingTask()
+ .getEnvironment()
+ .getTaskManagerInfo()
+ .getConfiguration()
+ .getLong(MetricOptions.LATENCY_INTERVAL),
+ getOperatorID(),
+ getRuntimeContext().getIndexOfThisSubtask());
+
// restore the state if necessary.
final List<SplitT> splits = CollectionUtil.iterableToList(readerState.get());
if (!splits.isEmpty()) {
@@ -296,6 +318,7 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
sourceReader.start();
eventTimeLogic.startPeriodicWatermarkEmits();
+ latencyMarerEmitter.startLatencyMarkerEmit();
}
@Override
@@ -303,6 +326,9 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
if (eventTimeLogic != null) {
eventTimeLogic.stopPeriodicWatermarkEmits();
}
+ if (latencyMarerEmitter != null) {
+ latencyMarerEmitter.stopLatencyMarkerEmit();
+ }
super.finish();
finished.complete(null);
@@ -348,6 +374,7 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
switch (operatingMode) {
case OUTPUT_NOT_INITIALIZED:
currentMainOutput = eventTimeLogic.createMainOutput(output);
+ latencyMarerEmitter.emitMainOutput(output);
lastInvokedOutput = output;
this.operatingMode = OperatingMode.READING;
return convertToInternalStatus(sourceReader.pollNext(currentMainOutput));
@@ -478,4 +505,79 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
this.forcedStopFuture.complete(null);
}
}
+
+ private static class LatencyMarkerEmitter<OUT> {
+
+ private final ProcessingTimeService timeService;
+
+ private final long latencyTrackingInterval;
+
+ private final OperatorID operatorId;
+
+ private final int subtaskIndex;
+
+ @Nullable private DataOutput<OUT> currentMainOutput;
+
+ @Nullable private ScheduledFuture<?> latencyMarkerTimer;
+
+ public LatencyMarkerEmitter(
+ final ProcessingTimeService timeService,
+ long latencyTrackingInterval,
+ final OperatorID operatorId,
+ final int subtaskIndex) {
+ this.timeService = timeService;
+ this.latencyTrackingInterval = latencyTrackingInterval;
+ this.operatorId = operatorId;
+ this.subtaskIndex = subtaskIndex;
+ }
+
+ // ------------------------------------------------------------------------
+
+ public void emitMainOutput(PushingAsyncDataInput.DataOutput<OUT> output) {
+ // At the moment, we assume only one output is ever created!
+ // This assumption is strict, currently, because many of the classes in this
+ // implementation
+ // do not support re-assigning the underlying output
+ checkState(currentMainOutput == null, "Main output has already been set.");
+ currentMainOutput = output;
+ }
+
+ public void startLatencyMarkerEmit() {
+ checkState(
+ latencyMarkerTimer == null, "Latency marker emitter has already been started");
+ if (latencyTrackingInterval == 0) {
+ // a value of zero means not activated
+ return;
+ }
+ latencyMarkerTimer =
+ timeService.scheduleWithFixedDelay(
+ this::triggerLatencyMarkerEmit, 0L, latencyTrackingInterval);
+ }
+
+ public void stopLatencyMarkerEmit() {
+ if (latencyMarkerTimer != null) {
+ latencyMarkerTimer.cancel(false);
+ latencyMarkerTimer = null;
+ }
+ }
+
+ void triggerLatencyMarkerEmit(@SuppressWarnings("unused") long wallClockTimestamp) {
+ if (currentMainOutput != null) {
+ try {
+ // ProcessingTimeService callbacks are executed under the
+ // checkpointing lock
+ currentMainOutput.emitLatencyMarker(
+ new LatencyMarker(
+ timeService.getCurrentProcessingTime(),
+ operatorId,
+ subtaskIndex));
+ } catch (Throwable t) {
+ // we catch the Throwable here so that we don't trigger the
+ // processing
+ // timer services async exception handler
+ LOG.warn("Error while emitting latency marker.", t);
+ }
+ }
+ }
+ }
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/SourceOperatorEventTimeTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/SourceOperatorEventTimeTest.java
index 39f5de4..c0a84f9 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/SourceOperatorEventTimeTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/SourceOperatorEventTimeTest.java
@@ -18,22 +18,30 @@
package org.apache.flink.streaming.api.operators.source;
+import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.eventtime.Watermark;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.state.OperatorStateStore;
import org.apache.flink.api.connector.source.ReaderOutput;
import org.apache.flink.api.connector.source.SourceReader;
import org.apache.flink.api.connector.source.mocks.MockSourceSplit;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.CloseableRegistry;
import org.apache.flink.core.io.InputStatus;
import org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder;
+import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider;
import org.apache.flink.runtime.state.StateInitializationContext;
import org.apache.flink.runtime.state.StateInitializationContextImpl;
+import org.apache.flink.runtime.state.TestTaskStateManager;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.streaming.api.operators.SourceOperator;
import org.apache.flink.streaming.runtime.io.DataInputStatus;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
+import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask;
+import org.apache.flink.streaming.runtime.tasks.StreamMockEnvironment;
import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService;
+import org.apache.flink.streaming.util.MockOutput;
+import org.apache.flink.streaming.util.MockStreamConfig;
import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
@@ -243,6 +251,19 @@ public class SourceOperatorEventTimeTest {
final SourceOperator<T, MockSourceSplit> sourceOperator =
new TestingSourceOperator<>(
reader, watermarkStrategy, timeService, emitProgressiveWatermarks);
+
+ sourceOperator.setup(
+ new SourceOperatorStreamTask<Integer>(
+ new StreamMockEnvironment(
+ new Configuration(),
+ new Configuration(),
+ new ExecutionConfig(),
+ 1L,
+ new MockInputSplitProvider(),
+ 1,
+ new TestTaskStateManager())),
+ new MockStreamConfig(new Configuration(), 1),
+ new MockOutput<>(new ArrayList<>()));
sourceOperator.initializeState(stateContext);
sourceOperator.open();
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/SourceOperatorLatencyMetricsTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/SourceOperatorLatencyMetricsTest.java
new file mode 100644
index 0000000..7685927
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/SourceOperatorLatencyMetricsTest.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.runtime.operators;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.state.OperatorStateStore;
+import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
+import org.apache.flink.api.connector.source.ReaderOutput;
+import org.apache.flink.api.connector.source.mocks.MockSourceReader;
+import org.apache.flink.api.connector.source.mocks.MockSourceSplit;
+import org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.MetricOptions;
+import org.apache.flink.core.fs.CloseableRegistry;
+import org.apache.flink.core.io.InputStatus;
+import org.apache.flink.core.io.SimpleVersionedSerialization;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.runtime.operators.coordination.MockOperatorEventGateway;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
+import org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder;
+import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider;
+import org.apache.flink.runtime.state.AbstractStateBackend;
+import org.apache.flink.runtime.state.StateInitializationContext;
+import org.apache.flink.runtime.state.StateInitializationContextImpl;
+import org.apache.flink.runtime.state.TestTaskStateManager;
+import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
+import org.apache.flink.runtime.util.TestingTaskManagerRuntimeInfo;
+import org.apache.flink.streaming.api.graph.StreamConfig;
+import org.apache.flink.streaming.api.operators.SourceOperator;
+import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
+import org.apache.flink.streaming.runtime.tasks.OperatorChain;
+import org.apache.flink.streaming.runtime.tasks.RegularOperatorChain;
+import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask;
+import org.apache.flink.streaming.runtime.tasks.StreamMockEnvironment;
+import org.apache.flink.streaming.runtime.tasks.StreamTask;
+import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService;
+import org.apache.flink.streaming.util.CollectorDataOutput;
+import org.apache.flink.streaming.util.MockOutput;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/** Tests for the emission of latency markers by {@link SourceOperator} operators. */
+public class SourceOperatorLatencyMetricsTest extends TestLogger {
+
+ private static final long maxProcessingTime = 100L;
+ private static final long latencyMarkInterval = 10L;
+
+ /** Verifies that by default no latency metrics are emitted. */
+ @Test
+ public void testLatencyMarkEmissionDisabled() throws Exception {
+ testLatencyMarkEmission(
+ 0,
+ operator ->
+ setupSourceOperator(
+ operator,
+ new ExecutionConfig(),
+ MockEnvironment.builder().build()));
+ }
+
+ /** Verifies that latency metrics can be enabled via the {@link ExecutionConfig}. */
+ @Test
+ public void testLatencyMarkEmissionEnabledViaExecutionConfig() throws Exception {
+ testLatencyMarkEmission(
+ (int) (maxProcessingTime / latencyMarkInterval) + 1,
+ operator -> {
+ ExecutionConfig executionConfig = new ExecutionConfig();
+ executionConfig.setLatencyTrackingInterval(latencyMarkInterval);
+ setupSourceOperator(
+ operator, executionConfig, MockEnvironment.builder().build());
+ });
+ }
+
+ /** Verifies that latency metrics can be enabled via the configuration. */
+ @Test
+ public void testLatencyMarkEmissionEnabledViaFlinkConfig() throws Exception {
+ testLatencyMarkEmission(
+ (int) (maxProcessingTime / latencyMarkInterval) + 1,
+ operator -> {
+ Configuration tmConfig = new Configuration();
+ tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);
+ Environment env =
+ MockEnvironment.builder()
+ .setTaskManagerRuntimeInfo(
+ new TestingTaskManagerRuntimeInfo(tmConfig))
+ .build();
+ setupSourceOperator(operator, new ExecutionConfig(), env);
+ });
+ }
+
+ /**
+ * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are
+ * disabled via the configuration.
+ */
+ @Test
+ public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
+ testLatencyMarkEmission(
+ (int) (maxProcessingTime / latencyMarkInterval) + 1,
+ operator -> {
+ ExecutionConfig executionConfig = new ExecutionConfig();
+ executionConfig.setLatencyTrackingInterval(latencyMarkInterval);
+ Configuration tmConfig = new Configuration();
+ tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);
+ Environment env =
+ MockEnvironment.builder()
+ .setTaskManagerRuntimeInfo(
+ new TestingTaskManagerRuntimeInfo(tmConfig))
+ .build();
+ setupSourceOperator(operator, executionConfig, env);
+ });
+ }
+
+ /**
+ * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they
+ * are enabled via the configuration.
+ */
+ @Test
+ public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
+ testLatencyMarkEmission(
+ 0,
+ operator -> {
+ Configuration tmConfig = new Configuration();
+ tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);
+ Environment env =
+ MockEnvironment.builder()
+ .setTaskManagerRuntimeInfo(
+ new TestingTaskManagerRuntimeInfo(tmConfig))
+ .build();
+ ExecutionConfig executionConfig = new ExecutionConfig();
+ executionConfig.setLatencyTrackingInterval(0);
+ setupSourceOperator(operator, executionConfig, env);
+ });
+ }
+
+ private interface OperatorSetupOperation {
+ void setupSourceOperator(SourceOperator<Integer, ?> operator);
+ }
+
+ private void testLatencyMarkEmission(
+ int numberLatencyMarkers, OperatorSetupOperation operatorSetup) throws Exception {
+ final List<StreamElement> output = new ArrayList<>();
+
+ final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
+ testProcessingTimeService.setCurrentTime(0L);
+ final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);
+
+ // regular source operator
+ final SourceOperator<Integer, MockSourceSplit> sourceOperator =
+ new SourceOperator<>(
+ (context) ->
+ new ProcessingTimeServiceSourceReader(
+ testProcessingTimeService, processingTimes),
+ new MockOperatorEventGateway(),
+ new MockSourceSplitSerializer(),
+ WatermarkStrategy.noWatermarks(),
+ testProcessingTimeService,
+ new Configuration(),
+ "localhost",
+ true /* emit progressive watermarks */);
+ operatorSetup.setupSourceOperator(sourceOperator);
+
+ // run and wait to be stopped
+ OperatorChain<?, ?> operatorChain =
+ new RegularOperatorChain<>(
+ sourceOperator.getContainingTask(),
+ StreamTask.createRecordWriterDelegate(
+ sourceOperator.getOperatorConfig(),
+ new MockEnvironmentBuilder().build()));
+ try {
+ sourceOperator.initializeState(getStateContext());
+ sourceOperator.open();
+ sourceOperator.emitNext(new CollectorDataOutput<>(output));
+ sourceOperator.finish();
+ } finally {
+ operatorChain.close();
+ }
+
+ assertEquals(numberLatencyMarkers, output.size());
+
+ long timestamp = 0L;
+ int expectedLatencyIndex = 0;
+
+ int index = 0;
+ // verify that its only latency markers
+ for (; index < numberLatencyMarkers; index++) {
+ StreamElement streamElement = output.get(index);
+ assertTrue(streamElement.isLatencyMarker());
+ assertEquals(
+ sourceOperator.getOperatorID(),
+ streamElement.asLatencyMarker().getOperatorId());
+ assertEquals(0, streamElement.asLatencyMarker().getSubtaskIndex());
+
+ // determines the next latency mark that should've been emitted
+ // latency marks are emitted once per latencyMarkInterval,
+ // as a result of which we never emit both 10 and 11
+ while (timestamp > processingTimes.get(expectedLatencyIndex)) {
+ expectedLatencyIndex++;
+ }
+ assertEquals(
+ processingTimes.get(expectedLatencyIndex).longValue(),
+ streamElement.asLatencyMarker().getMarkedTime());
+
+ timestamp += latencyMarkInterval;
+ }
+ }
+
+ // ---------------- helper methods -------------------------
+
+ private static <T> void setupSourceOperator(
+ SourceOperator<T, ?> sourceOperator,
+ ExecutionConfig executionConfig,
+ Environment environment) {
+ StreamConfig streamConfig = new StreamConfig(new Configuration());
+ streamConfig.setOperatorID(new OperatorID());
+ try {
+ sourceOperator.setup(
+ new SourceOperatorStreamTask<Integer>(
+ getTestingEnvironment(executionConfig, environment)),
+ streamConfig,
+ new MockOutput<>(new ArrayList<>()));
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private StateInitializationContext getStateContext() throws Exception {
+ // Create a mock split.
+ byte[] serializedSplitWithVersion =
+ SimpleVersionedSerialization.writeVersionAndSerialize(
+ new MockSourceSplitSerializer(), new MockSourceSplit(1234, 10));
+
+ // Crate the state context.
+ OperatorStateStore operatorStateStore = createOperatorStateStore();
+ StateInitializationContext stateContext =
+ new StateInitializationContextImpl(null, operatorStateStore, null, null, null);
+
+ // Update the context.
+ stateContext
+ .getOperatorStateStore()
+ .getListState(
+ new ListStateDescriptor<>(
+ "SourceReaderState", BytePrimitiveArraySerializer.INSTANCE))
+ .update(Collections.singletonList(serializedSplitWithVersion));
+
+ return stateContext;
+ }
+
+ private OperatorStateStore createOperatorStateStore() throws Exception {
+ MockEnvironment env = new MockEnvironmentBuilder().build();
+ final AbstractStateBackend abstractStateBackend = new HashMapStateBackend();
+ CloseableRegistry cancelStreamRegistry = new CloseableRegistry();
+ return abstractStateBackend.createOperatorStateBackend(
+ env, "test-operator", Collections.emptyList(), cancelStreamRegistry);
+ }
+
+ private static Environment getTestingEnvironment(
+ ExecutionConfig executionConfig, Environment environment) {
+ StreamMockEnvironment mockEnvironment =
+ new StreamMockEnvironment(
+ new Configuration(),
+ new Configuration(),
+ executionConfig,
+ 1L,
+ new MockInputSplitProvider(),
+ 1,
+ new TestTaskStateManager());
+ mockEnvironment.setTaskManagerInfo(environment.getTaskManagerInfo());
+ return mockEnvironment;
+ }
+
+ // ------------------------------------------------------------------------
+
+ private static final class ProcessingTimeServiceSourceReader extends MockSourceReader {
+
+ private final TestProcessingTimeService processingTimeService;
+ private final List<Long> processingTimes;
+
+ private boolean closed = false;
+
+ private ProcessingTimeServiceSourceReader(
+ TestProcessingTimeService processingTimeService, List<Long> processingTimes) {
+ this.processingTimeService = processingTimeService;
+ this.processingTimes = processingTimes;
+ }
+
+ @Override
+ public InputStatus pollNext(ReaderOutput<Integer> sourceOutput) throws Exception {
+ for (Long processingTime : processingTimes) {
+ if (closed) {
+ break;
+ }
+ processingTimeService.setCurrentTime(processingTime);
+ }
+ return super.pollNext(sourceOutput);
+ }
+
+ @Override
+ public void close() throws Exception {
+ closed = true;
+ super.close();
+ }
+ }
+}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorDataOutput.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorDataOutput.java
new file mode 100644
index 0000000..2fb2ee5
--- /dev/null
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/CollectorDataOutput.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util;
+
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput;
+import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
+import org.apache.flink.streaming.runtime.streamrecord.StreamElement;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.util.List;
+
+/** Collecting {@link DataOutput} for {@link StreamRecord}. */
+public class CollectorDataOutput<T> implements DataOutput<T> {
+
+ private final List<StreamElement> streamElements;
+
+ public CollectorDataOutput(List<StreamElement> streamElements) {
+ this.streamElements = streamElements;
+ }
+
+ @Override
+ public void emitRecord(StreamRecord<T> streamRecord) throws Exception {
+ try {
+ ClassLoader classLoader = streamRecord.getClass().getClassLoader();
+ T copied =
+ InstantiationUtil.deserializeObject(
+ InstantiationUtil.serializeObject(streamRecord.getValue()),
+ classLoader);
+ streamElements.add(streamRecord.copy(copied));
+ } catch (IOException | ClassNotFoundException ex) {
+ throw new RuntimeException("Unable to deserialize record: " + streamRecord, ex);
+ }
+ }
+
+ @Override
+ public void emitWatermark(Watermark watermark) throws Exception {
+ streamElements.add(watermark);
+ }
+
+ @Override
+ public void emitWatermarkStatus(WatermarkStatus watermarkStatus) throws Exception {
+ streamElements.add(watermarkStatus);
+ }
+
+ @Override
+ public void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception {
+ streamElements.add(latencyMarker);
+ }
+}