You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2021/10/15 08:09:14 UTC

[flink] 01/04: [FLINK-23704][task] Extract LatencyMarkerEmitter class

This is an automated email from the ASF dual-hosted git repository.

pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 32cca5603036b1c1aaa8426daccc7516a9309a5c
Author: Piotr Nowojski <pi...@gmail.com>
AuthorDate: Thu Oct 14 16:28:43 2021 +0200

    [FLINK-23704][task] Extract LatencyMarkerEmitter class
---
 .../api/operators/LatencyMarkerEmitter.java        | 71 ++++++++++++++++++++++
 .../streaming/api/operators/StreamSource.java      | 53 +---------------
 2 files changed, 74 insertions(+), 50 deletions(-)

diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/LatencyMarkerEmitter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/LatencyMarkerEmitter.java
new file mode 100644
index 0000000..cbd677a
--- /dev/null
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/LatencyMarkerEmitter.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
+
+import java.util.concurrent.ScheduledFuture;
+
+/** A small helper class to periodically emit {@link LatencyMarker}. */
+@Internal
+class LatencyMarkerEmitter<OUT> {
+    private final ScheduledFuture<?> latencyMarkTimer;
+
+    public LatencyMarkerEmitter(
+            ProcessingTimeService processingTimeService,
+            EmitAction emitAction,
+            long latencyTrackingInterval,
+            OperatorID operatorId,
+            int subtaskIndex) {
+        latencyMarkTimer =
+                processingTimeService.scheduleWithFixedDelay(
+                        new ProcessingTimeCallback() {
+                            @Override
+                            public void onProcessingTime(long timestamp) {
+                                try {
+                                    emitAction.emitLatencyMarker(
+                                            new LatencyMarker(
+                                                    processingTimeService
+                                                            .getCurrentProcessingTime(),
+                                                    operatorId,
+                                                    subtaskIndex));
+                                } catch (Throwable t) {
+                                    // we catch the Throwables here so that we don't trigger the
+                                    // processing
+                                    // timer services async exception handler
+                                    AbstractStreamOperator.LOG.warn(
+                                            "Error while emitting latency marker.", t);
+                                }
+                            }
+                        },
+                        0L,
+                        latencyTrackingInterval);
+    }
+
+    public void close() {
+        latencyMarkTimer.cancel(true);
+    }
+
+    interface EmitAction {
+        void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception;
+    }
+}
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
index 833d2a4..f7f4007 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamSource.java
@@ -21,16 +21,10 @@ import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.MetricOptions;
-import org.apache.flink.runtime.jobgraph.OperatorID;
 import org.apache.flink.streaming.api.TimeCharacteristic;
 import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker;
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
 import org.apache.flink.streaming.runtime.tasks.OperatorChain;
-import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
-import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
-
-import java.util.concurrent.ScheduledFuture;
 
 /**
  * {@link StreamOperator} for streaming sources.
@@ -88,12 +82,12 @@ public class StreamSource<OUT, SRC extends SourceFunction<OUT>>
                         ? getExecutionConfig().getLatencyTrackingInterval()
                         : configuration.getLong(MetricOptions.LATENCY_INTERVAL);
 
-        LatencyMarksEmitter<OUT> latencyEmitter = null;
+        LatencyMarkerEmitter<OUT> latencyEmitter = null;
         if (latencyTrackingInterval > 0) {
             latencyEmitter =
-                    new LatencyMarksEmitter<>(
+                    new LatencyMarkerEmitter<>(
                             getProcessingTimeService(),
-                            collector,
+                            collector::emitLatencyMarker,
                             latencyTrackingInterval,
                             this.getOperatorID(),
                             getRuntimeContext().getIndexOfThisSubtask());
@@ -165,45 +159,4 @@ public class StreamSource<OUT, SRC extends SourceFunction<OUT>>
     protected boolean isCanceledOrStopped() {
         return canceledOrStopped;
     }
-
-    private static class LatencyMarksEmitter<OUT> {
-        private final ScheduledFuture<?> latencyMarkTimer;
-
-        public LatencyMarksEmitter(
-                final ProcessingTimeService processingTimeService,
-                final Output<StreamRecord<OUT>> output,
-                long latencyTrackingInterval,
-                final OperatorID operatorId,
-                final int subtaskIndex) {
-
-            latencyMarkTimer =
-                    processingTimeService.scheduleWithFixedDelay(
-                            new ProcessingTimeCallback() {
-                                @Override
-                                public void onProcessingTime(long timestamp) throws Exception {
-                                    try {
-                                        // ProcessingTimeService callbacks are executed under the
-                                        // checkpointing lock
-                                        output.emitLatencyMarker(
-                                                new LatencyMarker(
-                                                        processingTimeService
-                                                                .getCurrentProcessingTime(),
-                                                        operatorId,
-                                                        subtaskIndex));
-                                    } catch (Throwable t) {
-                                        // we catch the Throwables here so that we don't trigger the
-                                        // processing
-                                        // timer services async exception handler
-                                        LOG.warn("Error while emitting latency marker.", t);
-                                    }
-                                }
-                            },
-                            0L,
-                            latencyTrackingInterval);
-        }
-
-        public void close() {
-            latencyMarkTimer.cancel(true);
-        }
-    }
 }