You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ar...@apache.org on 2021/08/27 18:29:51 UTC
[flink] branch master updated: [FLINK-23776][datastream] Optimize
source metric calculation
This is an automated email from the ASF dual-hosted git repository.
arvid pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new fd429d0 [FLINK-23776][datastream] Optimize source metric calculation
fd429d0 is described below
commit fd429d084264357d3cfbfd8b2b62cf8327a8fd79
Author: Arvid Heise <ar...@ververica.com>
AuthorDate: Wed Aug 25 21:11:50 2021 +0200
[FLINK-23776][datastream] Optimize source metric calculation
There is a performance regression of 15% on new sources in InputBenchmark. A couple of issues are fixed with this commit:
- MetricTrackingOutput in SourceOperator adds 1 virtual calls per record. This accounts for ~5% of the regression. The solution is to move the functionality inside WatermarkToDataOutput.
- Lazy registration of CURRENT_EMIT_EVENT_TIME_LAG gauge. This adds ~2% overhead as it checks for each record whether the metric has been registered already. The solution is to always add the metric and just return some UNDEFINED value when there is no record.
- Check of NO_TIMESTAMP. This adds ~3% overhead as it's a long comparison for each record. The solution is to lazily check the timestamp in the gauge. This solution depends on the prior solution.
- Fusion of recordEmitted() and eventTimeEmitted(). This removes 2% overhead and is possible after the previous optimizations.
A bit of regression may remain after this commit. Future work may further improve the situation. However, it may also be the price to be paid for the additional metrics. In real sources with I/O, the remaining regression will not be as noticeable. For example, the micro benchmarks imply that the new source is 2x faster than the old sources but we do not see any difference in actual sources.
---
.../base/source/reader/SourceMetricsITCase.java | 9 +-
.../groups/InternalSourceReaderMetricGroup.java | 110 ++++++++++++---------
.../streaming/api/operators/SourceOperator.java | 81 +++++----------
.../io/StreamMultipleInputProcessorFactory.java | 17 +++-
.../streaming/runtime/tasks/OperatorChain.java | 5 +-
.../runtime/tasks/SourceOperatorStreamTask.java | 23 +++--
.../operators/source/TestingSourceOperator.java | 1 +
7 files changed, 128 insertions(+), 118 deletions(-)
diff --git a/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/SourceMetricsITCase.java b/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/SourceMetricsITCase.java
index 399fe96..508e1ef 100644
--- a/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/SourceMetricsITCase.java
+++ b/flink-connectors/flink-connector-base/src/test/java/org/apache/flink/connector/base/source/reader/SourceMetricsITCase.java
@@ -32,6 +32,7 @@ import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.Metric;
import org.apache.flink.metrics.groups.OperatorMetricGroup;
import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.runtime.metrics.groups.InternalSourceReaderMetricGroup;
import org.apache.flink.runtime.testutils.InMemoryReporter;
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
import org.apache.flink.streaming.api.datastream.DataStream;
@@ -184,7 +185,9 @@ public class SourceMetricsITCase extends TestLogger {
// there are only 2 splits assigned; so two groups will not update metrics
if (group.getIOMetricGroup().getNumRecordsInCounter().getCount() == 0) {
// assert that optional metrics are not initialized when no split assigned
- assertThat(metrics.get(MetricNames.CURRENT_EMIT_EVENT_TIME_LAG), nullValue());
+ assertThat(
+ metrics.get(MetricNames.CURRENT_EMIT_EVENT_TIME_LAG),
+ isGauge(equalTo(InternalSourceReaderMetricGroup.UNDEFINED)));
assertThat(metrics.get(MetricNames.WATERMARK_LAG), nullValue());
continue;
}
@@ -224,7 +227,9 @@ public class SourceMetricsITCase extends TestLogger {
assertThat(watermarkLag, isCloseTo(WATERMARK_LAG, WATERMARK_EPSILON));
} else {
// assert that optional metrics are not initialized when no timestamp assigned
- assertThat(metrics.get(MetricNames.CURRENT_EMIT_EVENT_TIME_LAG), nullValue());
+ assertThat(
+ metrics.get(MetricNames.CURRENT_EMIT_EVENT_TIME_LAG),
+ isGauge(equalTo(InternalSourceReaderMetricGroup.UNDEFINED)));
assertThat(metrics.get(MetricNames.WATERMARK_LAG), nullValue());
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/InternalSourceReaderMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/InternalSourceReaderMetricGroup.java
index eaa1cd6..32762b4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/InternalSourceReaderMetricGroup.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/InternalSourceReaderMetricGroup.java
@@ -20,6 +20,8 @@ package org.apache.flink.runtime.metrics.groups;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.common.eventtime.TimestampAssigner;
+import org.apache.flink.api.common.eventtime.Watermark;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.MetricGroup;
@@ -35,17 +37,17 @@ import org.apache.flink.util.clock.SystemClock;
@Internal
public class InternalSourceReaderMetricGroup extends ProxyMetricGroup<MetricGroup>
implements SourceReaderMetricGroup {
-
- public static final long ACTIVE = Long.MAX_VALUE;
+ public static final long UNDEFINED = -1L;
+ private static final long ACTIVE = Long.MAX_VALUE;
+ private static final long MAX_WATERMARK_TIMESTAMP = Watermark.MAX_WATERMARK.getTimestamp();
private final OperatorIOMetricGroup operatorIOMetricGroup;
- private final Clock clock;
private final Counter numRecordsInErrors;
- private boolean watermarkLagRegistered;
- private boolean eventTimeLagRegistered;
+ private final Clock clock;
private long lastWatermark;
- private long lastEventTime;
+ private long lastEventTime = TimestampAssigner.NO_TIMESTAMP;
private long idleStartTime = ACTIVE;
+ private boolean firstWatermark = true;
private InternalSourceReaderMetricGroup(
MetricGroup parentMetricGroup,
@@ -55,9 +57,8 @@ public class InternalSourceReaderMetricGroup extends ProxyMetricGroup<MetricGrou
numRecordsInErrors = parentMetricGroup.counter(MetricNames.NUM_RECORDS_IN_ERRORS);
this.operatorIOMetricGroup = operatorIOMetricGroup;
this.clock = clock;
- parentMetricGroup.gauge(
- MetricNames.SOURCE_IDLE_TIME,
- () -> isIdling() ? this.clock.absoluteTimeMillis() - idleStartTime : 0);
+ parentMetricGroup.gauge(MetricNames.SOURCE_IDLE_TIME, this::getIdleTime);
+ parentMetricGroup.gauge(MetricNames.CURRENT_EMIT_EVENT_TIME_LAG, this::getEmitTimeLag);
}
public static InternalSourceReaderMetricGroup wrap(OperatorMetricGroup operatorMetricGroup) {
@@ -75,66 +76,85 @@ public class InternalSourceReaderMetricGroup extends ProxyMetricGroup<MetricGrou
SystemClock.getInstance());
}
- private boolean isIdling() {
- return idleStartTime != ACTIVE;
+ @Override
+ public Counter getNumRecordsInErrorsCounter() {
+ return numRecordsInErrors;
}
- public void idlingStarted() {
- if (!isIdling()) {
- idleStartTime = clock.absoluteTimeMillis();
- }
+ @Override
+ public void setPendingBytesGauge(Gauge<Long> pendingBytesGauge) {
+ gauge(MetricNames.PENDING_BYTES, pendingBytesGauge);
}
- public void recordEmitted() {
- idleStartTime = ACTIVE;
+ @Override
+ public void setPendingRecordsGauge(Gauge<Long> pendingRecordsGauge) {
+ gauge(MetricNames.PENDING_RECORDS, pendingRecordsGauge);
}
@Override
- public Counter getNumRecordsInErrorsCounter() {
- return numRecordsInErrors;
+ public OperatorIOMetricGroup getIOMetricGroup() {
+ return operatorIOMetricGroup;
}
+ /**
+ * Called when a new record was emitted with the given timestamp. {@link
+ * TimestampAssigner#NO_TIMESTAMP} should be indicated that the record did not have a timestamp.
+ *
+ * <p>Note this function should be called before the actual record is emitted such that chained
+ * processing does not influence the statistics.
+ */
+ public void recordEmitted(long timestamp) {
+ idleStartTime = ACTIVE;
+ lastEventTime = timestamp;
+ }
+
+ public void idlingStarted() {
+ if (!isIdling()) {
+ idleStartTime = clock.absoluteTimeMillis();
+ }
+ }
+
+ /**
+ * Called when a watermark was emitted.
+ *
+ * <p>Note this function should be called before the actual watermark is emitted such that
+ * chained processing does not influence the statistics.
+ */
public void watermarkEmitted(long watermark) {
+ if (watermark == MAX_WATERMARK_TIMESTAMP) {
+ return;
+ }
lastWatermark = watermark;
- // iff a respective source emits a watermark, Flink can provide the watermark lag
- if (!watermarkLagRegistered) {
- parentMetricGroup.gauge(
- MetricNames.WATERMARK_LAG, () -> clock.absoluteTimeMillis() - lastWatermark);
- watermarkLagRegistered = true;
+ if (firstWatermark) {
+ parentMetricGroup.gauge(MetricNames.WATERMARK_LAG, this::getWatermarkLag);
+ firstWatermark = false;
}
}
- public void eventTimeEmitted(long timestamp) {
- lastEventTime = timestamp;
- // iff a respective source emits a timestamp, Flink can provide the event lag
- if (!eventTimeLagRegistered) {
- parentMetricGroup.gauge(
- MetricNames.CURRENT_EMIT_EVENT_TIME_LAG,
- () -> getLastEmitTime() - lastEventTime);
- eventTimeLagRegistered = true;
- }
+ boolean isIdling() {
+ return idleStartTime != ACTIVE;
+ }
+
+ long getIdleTime() {
+ return isIdling() ? this.clock.absoluteTimeMillis() - idleStartTime : 0;
}
/**
- * This is a rough approximation. If the source is busy, we assume that <code>emit time == now()
+ * This is a rough approximation. If the source is busy, we assume that <code>
+ * emit time == now()
* </code>. If it's idling, we just take the time it started idling as the last emit time.
*/
private long getLastEmitTime() {
return isIdling() ? idleStartTime : clock.absoluteTimeMillis();
}
- @Override
- public void setPendingBytesGauge(Gauge<Long> pendingBytesGauge) {
- gauge(MetricNames.PENDING_BYTES, pendingBytesGauge);
+ long getEmitTimeLag() {
+ return lastEventTime != TimestampAssigner.NO_TIMESTAMP
+ ? getLastEmitTime() - lastEventTime
+ : UNDEFINED;
}
- @Override
- public void setPendingRecordsGauge(Gauge<Long> pendingRecordsGauge) {
- gauge(MetricNames.PENDING_RECORDS, pendingRecordsGauge);
- }
-
- @Override
- public OperatorIOMetricGroup getIOMetricGroup() {
- return operatorIOMetricGroup;
+ long getWatermarkLag() {
+ return getLastEmitTime() - lastWatermark;
}
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
index 60f00d3..bcfb639 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceOperator.java
@@ -19,7 +19,6 @@ package org.apache.flink.streaming.api.operators;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
-import org.apache.flink.api.common.eventtime.TimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;
@@ -45,15 +44,14 @@ import org.apache.flink.runtime.source.event.RequestSplitEvent;
import org.apache.flink.runtime.source.event.SourceEventWrapper;
import org.apache.flink.runtime.state.StateInitializationContext;
import org.apache.flink.runtime.state.StateSnapshotContext;
+import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.source.TimestampsAndWatermarks;
import org.apache.flink.streaming.api.operators.util.SimpleVersionedListState;
-import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.io.DataInputStatus;
import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput;
-import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
-import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
+import org.apache.flink.streaming.runtime.tasks.StreamTask;
import org.apache.flink.util.CollectionUtil;
import org.apache.flink.util.FlinkRuntimeException;
import org.apache.flink.util.UserCodeClassLoader;
@@ -172,6 +170,20 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
this.operatingMode = OperatingMode.OUTPUT_NOT_INITIALIZED;
}
+ @Override
+ public void setup(
+ StreamTask<?, ?> containingTask,
+ StreamConfig config,
+ Output<StreamRecord<OUT>> output) {
+ super.setup(containingTask, config, output);
+ initSourceMetricGroup();
+ }
+
+ @VisibleForTesting
+ protected void initSourceMetricGroup() {
+ sourceMetricGroup = InternalSourceReaderMetricGroup.wrap(getMetricGroup());
+ }
+
/**
* Initializes the reader. The code from this method should ideally happen in the constructor or
* in the operator factory even. It has to happen here at a slightly later stage, because of the
@@ -188,7 +200,6 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
if (sourceReader != null) {
return;
}
- sourceMetricGroup = InternalSourceReaderMetricGroup.wrap(getMetricGroup());
final int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
@@ -247,6 +258,10 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
sourceReader = readerFactory.apply(context);
}
+ public InternalSourceReaderMetricGroup getSourceMetricGroup() {
+ return sourceMetricGroup;
+ }
+
@Override
public void open() throws Exception {
initReader();
@@ -275,6 +290,7 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
// Register the reader to the coordinator.
registerReader();
+ sourceMetricGroup.idlingStarted();
// Start the reader after registration, sending messages in start is allowed.
sourceReader.start();
@@ -322,7 +338,7 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
// short circuit the hot path. Without this short circuit (READING handled in the
// switch/case) InputBenchmark.mapSink was showing a performance regression.
if (operatingMode == OperatingMode.READING) {
- return convertToInternalStatus(pollNext());
+ return convertToInternalStatus(sourceReader.pollNext(currentMainOutput));
}
return emitNextNotReading(output);
}
@@ -330,16 +346,16 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
private DataInputStatus emitNextNotReading(DataOutput<OUT> output) throws Exception {
switch (operatingMode) {
case OUTPUT_NOT_INITIALIZED:
- currentMainOutput =
- eventTimeLogic.createMainOutput(
- new MetricTrackingOutput<>(output, sourceMetricGroup));
+ currentMainOutput = eventTimeLogic.createMainOutput(output);
lastInvokedOutput = output;
this.operatingMode = OperatingMode.READING;
return convertToInternalStatus(sourceReader.pollNext(currentMainOutput));
case SOURCE_STOPPED:
this.operatingMode = OperatingMode.DATA_FINISHED;
+ sourceMetricGroup.idlingStarted();
return DataInputStatus.END_OF_DATA;
case DATA_FINISHED:
+ sourceMetricGroup.idlingStarted();
return DataInputStatus.END_OF_INPUT;
case READING:
default:
@@ -352,23 +368,17 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
case MORE_AVAILABLE:
return DataInputStatus.MORE_AVAILABLE;
case NOTHING_AVAILABLE:
+ sourceMetricGroup.idlingStarted();
return DataInputStatus.NOTHING_AVAILABLE;
case END_OF_INPUT:
this.operatingMode = OperatingMode.DATA_FINISHED;
+ sourceMetricGroup.idlingStarted();
return DataInputStatus.END_OF_DATA;
default:
throw new IllegalArgumentException("Unknown input status: " + inputStatus);
}
}
- private InputStatus pollNext() throws Exception {
- InputStatus inputStatus = sourceReader.pollNext(currentMainOutput);
- if (inputStatus == InputStatus.NOTHING_AVAILABLE) {
- sourceMetricGroup.idlingStarted();
- }
- return inputStatus;
- }
-
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
long checkpointId = context.getCheckpointId();
@@ -447,41 +457,4 @@ public class SourceOperator<OUT, SplitT extends SourceSplit> extends AbstractStr
ListState<SplitT> getReaderState() {
return readerState;
}
-
- private static class MetricTrackingOutput<OUT> implements DataOutput<OUT> {
- private final DataOutput<OUT> output;
- private final InternalSourceReaderMetricGroup sourceMetricGroup;
-
- public MetricTrackingOutput(
- DataOutput<OUT> output, InternalSourceReaderMetricGroup sourceMetricGroup) {
- this.output = output;
- this.sourceMetricGroup = sourceMetricGroup;
- }
-
- @Override
- public void emitRecord(StreamRecord<OUT> streamRecord) throws Exception {
- output.emitRecord(streamRecord);
- this.sourceMetricGroup.recordEmitted();
- long timestamp = streamRecord.getTimestamp();
- if (timestamp != TimestampAssigner.NO_TIMESTAMP) {
- this.sourceMetricGroup.eventTimeEmitted(timestamp);
- }
- }
-
- @Override
- public void emitWatermark(Watermark watermark) throws Exception {
- output.emitWatermark(watermark);
- this.sourceMetricGroup.watermarkEmitted(watermark.getTimestamp());
- }
-
- @Override
- public void emitWatermarkStatus(WatermarkStatus watermarkStatus) throws Exception {
- output.emitWatermarkStatus(watermarkStatus);
- }
-
- @Override
- public void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception {
- output.emitLatencyMarker(latencyMarker);
- }
- }
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamMultipleInputProcessorFactory.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamMultipleInputProcessorFactory.java
index fcfffff..9afa00e 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamMultipleInputProcessorFactory.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamMultipleInputProcessorFactory.java
@@ -31,6 +31,7 @@ import org.apache.flink.runtime.checkpoint.InflightDataRescalingDescriptor;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.jobgraph.tasks.TaskInvokable;
import org.apache.flink.runtime.memory.MemoryManager;
+import org.apache.flink.runtime.metrics.groups.InternalSourceReaderMetricGroup;
import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.Input;
@@ -211,14 +212,19 @@ public class StreamMultipleInputProcessorFactory {
} else if (configuredInput instanceof StreamConfig.SourceInputConfig) {
StreamConfig.SourceInputConfig sourceInput =
(StreamConfig.SourceInputConfig) configuredInput;
- WatermarkGaugeExposingOutput<StreamRecord<?>> chainedSourceOutput =
- operatorChain.getChainedSourceOutput(sourceInput);
+ OperatorChain.ChainedSource chainedSource =
+ operatorChain.getChainedSource(sourceInput);
inputProcessors[i] =
new StreamOneInputProcessor(
inputs[i],
new StreamTaskSourceOutput(
- chainedSourceOutput, inputWatermarkGauges[i]),
+ chainedSource.getSourceOutput(),
+ inputWatermarkGauges[i],
+ chainedSource
+ .getSourceTaskInput()
+ .getOperator()
+ .getSourceMetricGroup()),
operatorChain);
} else {
throw new UnsupportedOperationException("Unknown input type: " + configuredInput);
@@ -285,8 +291,9 @@ public class StreamMultipleInputProcessorFactory {
public StreamTaskSourceOutput(
WatermarkGaugeExposingOutput<StreamRecord<?>> chainedSourceOutput,
- WatermarkGauge inputWatermarkGauge) {
- super(chainedSourceOutput, new SimpleCounter(), inputWatermarkGauge);
+ WatermarkGauge inputWatermarkGauge,
+ InternalSourceReaderMetricGroup metricGroup) {
+ super(chainedSourceOutput, metricGroup, inputWatermarkGauge);
this.chainedOutput = chainedSourceOutput;
}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index 902a389..26380ab 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -369,13 +369,12 @@ public abstract class OperatorChain<OUT, OP extends StreamOperator<OUT>>
return mainOperatorOutput;
}
- public WatermarkGaugeExposingOutput<StreamRecord<?>> getChainedSourceOutput(
- StreamConfig.SourceInputConfig sourceInput) {
+ public ChainedSource getChainedSource(StreamConfig.SourceInputConfig sourceInput) {
checkArgument(
chainedSources.containsKey(sourceInput),
"Chained source with sourcedId = [%s] was not found",
sourceInput);
- return chainedSources.get(sourceInput).getSourceOutput();
+ return chainedSources.get(sourceInput);
}
public List<Output<StreamRecord<?>>> getChainedSourceOutputs() {
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTask.java
index a2658ae..ce9de79 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceOperatorStreamTask.java
@@ -27,6 +27,7 @@ import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.metrics.MetricNames;
+import org.apache.flink.runtime.metrics.groups.InternalSourceReaderMetricGroup;
import org.apache.flink.runtime.state.CheckpointStorageLocationReference;
import org.apache.flink.streaming.api.operators.Output;
import org.apache.flink.streaming.api.operators.SourceOperator;
@@ -87,14 +88,13 @@ public class SourceOperatorStreamTask<T> extends StreamTask<T, SourceOperator<T,
input = new StreamTaskSourceInput<>(sourceOperator, 0, 0);
}
- Counter numRecordsOut =
- sourceOperator.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
-
// The SourceOperatorStreamTask doesn't have any inputs, so there is no need for
// a WatermarkGauge on the input.
output =
- new AsyncDataOutputToOutput<>(
- operatorChain.getMainOperatorOutput(), numRecordsOut, null);
+ new AsyncDataOutputToOutput<T>(
+ operatorChain.getMainOperatorOutput(),
+ sourceOperator.getSourceMetricGroup(),
+ null);
inputProcessor = new StreamOneInputProcessor<>(input, output, operatorChain);
@@ -174,22 +174,25 @@ public class SourceOperatorStreamTask<T> extends StreamTask<T, SourceOperator<T,
public static class AsyncDataOutputToOutput<T> implements DataOutput<T> {
private final Output<StreamRecord<T>> output;
- private final Counter numRecordsOut;
+ private final InternalSourceReaderMetricGroup metricGroup;
@Nullable private final WatermarkGauge inputWatermarkGauge;
+ private final Counter numRecordsOut;
public AsyncDataOutputToOutput(
Output<StreamRecord<T>> output,
- Counter numRecordsOut,
+ InternalSourceReaderMetricGroup metricGroup,
@Nullable WatermarkGauge inputWatermarkGauge) {
this.output = checkNotNull(output);
- this.numRecordsOut = numRecordsOut;
+ this.numRecordsOut = metricGroup.getIOMetricGroup().getNumRecordsOutCounter();
this.inputWatermarkGauge = inputWatermarkGauge;
+ this.metricGroup = metricGroup;
}
@Override
public void emitRecord(StreamRecord<T> streamRecord) {
numRecordsOut.inc();
+ metricGroup.recordEmitted(streamRecord.getTimestamp());
output.collect(streamRecord);
}
@@ -200,9 +203,11 @@ public class SourceOperatorStreamTask<T> extends StreamTask<T, SourceOperator<T,
@Override
public void emitWatermark(Watermark watermark) {
+ long watermarkTimestamp = watermark.getTimestamp();
if (inputWatermarkGauge != null) {
- inputWatermarkGauge.setCurrentWatermark(watermark.getTimestamp());
+ inputWatermarkGauge.setCurrentWatermark(watermarkTimestamp);
}
+ metricGroup.watermarkEmitted(watermarkTimestamp);
output.emitWatermark(watermark);
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/TestingSourceOperator.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/TestingSourceOperator.java
index 7fda4dd..7becd40 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/TestingSourceOperator.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/source/TestingSourceOperator.java
@@ -95,6 +95,7 @@ public class TestingSourceOperator<T> extends SourceOperator<T, MockSourceSplit>
this.subtaskIndex = subtaskIndex;
this.parallelism = parallelism;
this.metrics = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
+ initSourceMetricGroup();
// unchecked wrapping is okay to keep tests simpler
try {