You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2024/11/04 17:03:40 UTC
(flink) 02/02: [FLINK-36624][checkpointing] Log JobID from the SourceCoordinator
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 96bb45d1f7ff81dde593a4cf9a36dddacfe6d762
Author: Piotr Nowojski <pi...@gmail.com>
AuthorDate: Mon Nov 4 11:13:56 2024 +0100
[FLINK-36624][checkpointing] Log JobID from the SourceCoordinator
---
.../source/coordinator/SourceCoordinator.java | 19 +++++++++++----
.../coordinator/SourceCoordinatorContext.java | 9 ++++++--
.../coordinator/SourceCoordinatorProvider.java | 2 ++
.../SourceCoordinatorAlignmentTest.java | 4 ++++
.../coordinator/SourceCoordinatorContextTest.java | 3 +++
.../source/coordinator/SourceCoordinatorTest.java | 6 +++++
.../coordinator/SourceCoordinatorTestBase.java | 3 +++
.../apache/flink/test/misc/JobIDLoggingITCase.java | 27 +++++++++++++++++++++-
8 files changed, 65 insertions(+), 8 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java
index 982310bc540..4a9c252a4a1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinator.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.source.coordinator;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.eventtime.Watermark;
import org.apache.flink.api.common.eventtime.WatermarkAlignmentParams;
import org.apache.flink.api.connector.source.DynamicFilteringInfo;
@@ -49,6 +50,7 @@ import org.apache.flink.runtime.state.heap.AbstractHeapPriorityQueueElement;
import org.apache.flink.runtime.state.heap.HeapPriorityQueue;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.MdcUtils;
import org.apache.flink.util.TemporaryClassLoaderContext;
import org.apache.flink.util.function.ThrowingRunnable;
@@ -103,6 +105,7 @@ public class SourceCoordinator<SplitT extends SourceSplit, EnumChkT>
private final WatermarkAlignmentParams watermarkAlignmentParams;
+ private final JobID jobID;
/** The name of the operator this SourceCoordinator is associated with. */
private final String operatorName;
/** The Source that is associated with this SourceCoordinator. */
@@ -133,6 +136,7 @@ public class SourceCoordinator<SplitT extends SourceSplit, EnumChkT>
SourceCoordinatorContext<SplitT> context,
CoordinatorStore coordinatorStore) {
this(
+ new JobID(),
operatorName,
source,
context,
@@ -142,12 +146,14 @@ public class SourceCoordinator<SplitT extends SourceSplit, EnumChkT>
}
public SourceCoordinator(
+ JobID jobID,
String operatorName,
Source<?, SplitT, EnumChkT> source,
SourceCoordinatorContext<SplitT> context,
CoordinatorStore coordinatorStore,
WatermarkAlignmentParams watermarkAlignmentParams,
@Nullable String coordinatorListeningID) {
+ this.jobID = jobID;
this.operatorName = operatorName;
this.source = source;
this.enumCheckpointSerializer = source.getEnumeratorCheckpointSerializer();
@@ -296,12 +302,15 @@ public class SourceCoordinator<SplitT extends SourceSplit, EnumChkT>
@Override
public void close() throws Exception {
- LOG.info("Closing SourceCoordinator for source {}.", operatorName);
- if (started) {
- closeQuietly(enumerator);
+ try (MdcUtils.MdcCloseable mdcCloseable =
+ MdcUtils.withContext(MdcUtils.asContextData(jobID))) {
+ LOG.info("Closing SourceCoordinator for source {}.", operatorName);
+ if (started) {
+ closeQuietly(enumerator);
+ }
+ closeQuietly(context);
+ LOG.info("Source coordinator for source {} closed.", operatorName);
}
- closeQuietly(context);
- LOG.info("Source coordinator for source {} closed.", operatorName);
}
@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContext.java
index 7965df05409..e543abf62ca 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContext.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.source.coordinator;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.connector.source.ReaderInfo;
import org.apache.flink.api.connector.source.SourceEvent;
import org.apache.flink.api.connector.source.SourceSplit;
@@ -41,6 +42,7 @@ import org.apache.flink.runtime.source.event.NoMoreSplitsEvent;
import org.apache.flink.runtime.source.event.SourceEventWrapper;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.MdcUtils;
import org.apache.flink.util.TernaryBoolean;
import org.apache.flink.util.ThrowableCatchingRunnable;
import org.apache.flink.util.concurrent.ExecutorThreadFactory;
@@ -120,12 +122,14 @@ public class SourceCoordinatorContext<SplitT extends SourceSplit>
private volatile TernaryBoolean backlog = TernaryBoolean.UNDEFINED;
public SourceCoordinatorContext(
+ JobID jobID,
SourceCoordinatorProvider.CoordinatorExecutorThreadFactory coordinatorThreadFactory,
int numWorkerThreads,
OperatorCoordinator.Context operatorCoordinatorContext,
SimpleVersionedSerializer<SplitT> splitSerializer,
boolean supportsConcurrentExecutionAttempts) {
this(
+ jobID,
Executors.newScheduledThreadPool(1, coordinatorThreadFactory),
Executors.newScheduledThreadPool(
numWorkerThreads,
@@ -141,6 +145,7 @@ public class SourceCoordinatorContext<SplitT extends SourceSplit>
// Package private method for unit test.
@VisibleForTesting
SourceCoordinatorContext(
+ JobID jobID,
ScheduledExecutorService coordinatorExecutor,
ScheduledExecutorService workerExecutor,
SourceCoordinatorProvider.CoordinatorExecutorThreadFactory coordinatorThreadFactory,
@@ -149,7 +154,7 @@ public class SourceCoordinatorContext<SplitT extends SourceSplit>
SplitAssignmentTracker<SplitT> splitAssignmentTracker,
boolean supportsConcurrentExecutionAttempts) {
this.workerExecutor = workerExecutor;
- this.coordinatorExecutor = coordinatorExecutor;
+ this.coordinatorExecutor = MdcUtils.scopeToJob(jobID, coordinatorExecutor);
this.coordinatorThreadFactory = coordinatorThreadFactory;
this.operatorCoordinatorContext = operatorCoordinatorContext;
this.splitSerializer = splitSerializer;
@@ -160,7 +165,7 @@ public class SourceCoordinatorContext<SplitT extends SourceSplit>
final Executor errorHandlingCoordinatorExecutor =
(runnable) ->
- coordinatorExecutor.execute(
+ this.coordinatorExecutor.execute(
new ThrowableCatchingRunnable(
this::handleUncaughtExceptionFromAsyncCall, runnable));
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProvider.java
index d238e28be5f..742a0bcb563 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProvider.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorProvider.java
@@ -82,12 +82,14 @@ public class SourceCoordinatorProvider<SplitT extends SourceSplit>
SimpleVersionedSerializer<SplitT> splitSerializer = source.getSplitSerializer();
SourceCoordinatorContext<SplitT> sourceCoordinatorContext =
new SourceCoordinatorContext<>(
+ context.getJobID(),
coordinatorThreadFactory,
numWorkerThreads,
context,
splitSerializer,
context.isConcurrentExecutionAttemptsSupported());
return new SourceCoordinator<>(
+ context.getJobID(),
operatorName,
source,
sourceCoordinatorContext,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorAlignmentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorAlignmentTest.java
index d479c34f63f..f0cba2b43f4 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorAlignmentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorAlignmentTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.source.coordinator;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.eventtime.Watermark;
import org.apache.flink.api.common.eventtime.WatermarkAlignmentParams;
import org.apache.flink.api.connector.source.Source;
@@ -166,6 +167,7 @@ class SourceCoordinatorAlignmentTest extends SourceCoordinatorTestBase {
AtomicInteger counter1 = new AtomicInteger(0);
sourceCoordinator =
new SourceCoordinator<MockSourceSplit, Set<MockSourceSplit>>(
+ new JobID(),
OPERATOR_NAME,
mockSource,
getNewSourceCoordinatorContext(),
@@ -184,6 +186,7 @@ class SourceCoordinatorAlignmentTest extends SourceCoordinatorTestBase {
CountDownLatch latch = new CountDownLatch(2);
sourceCoordinator =
new SourceCoordinator<MockSourceSplit, Set<MockSourceSplit>>(
+ new JobID(),
OPERATOR_NAME,
mockSource,
getNewSourceCoordinatorContext(),
@@ -218,6 +221,7 @@ class SourceCoordinatorAlignmentTest extends SourceCoordinatorTestBase {
CountDownLatch latch = new CountDownLatch(1);
sourceCoordinator =
new SourceCoordinator<MockSourceSplit, Set<MockSourceSplit>>(
+ new JobID(),
OPERATOR_NAME,
mockSource,
getNewSourceCoordinatorContext(),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContextTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContextTest.java
index 0d4c1a02d4f..4b92191d45b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContextTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorContextTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.source.coordinator;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.connector.source.ReaderInfo;
import org.apache.flink.api.connector.source.SplitsAssignment;
import org.apache.flink.api.connector.source.mocks.MockSourceSplit;
@@ -160,6 +161,7 @@ class SourceCoordinatorContextTest extends SourceCoordinatorTestBase {
new ManuallyTriggeredScheduledExecutorService();
SourceCoordinatorContext<MockSourceSplit> testingContext =
new SourceCoordinatorContext<>(
+ new JobID(),
coordinatorExecutorWithExceptionHandler,
manualWorkerExecutor,
new SourceCoordinatorProvider.CoordinatorExecutorThreadFactory(
@@ -195,6 +197,7 @@ class SourceCoordinatorContextTest extends SourceCoordinatorTestBase {
SourceCoordinatorContext<MockSourceSplit> testingContext =
new SourceCoordinatorContext<>(
+ new JobID(),
manualCoordinatorExecutor,
manualWorkerExecutor,
new SourceCoordinatorProvider.CoordinatorExecutorThreadFactory(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorTest.java
index 06343c87413..0bf8e526f36 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.source.coordinator;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.eventtime.WatermarkAlignmentParams;
import org.apache.flink.api.connector.source.Boundedness;
import org.apache.flink.api.connector.source.DynamicFilteringInfo;
@@ -290,6 +291,7 @@ class SourceCoordinatorTest extends SourceCoordinatorTestBase {
};
final SourceCoordinator<?, ?> coordinator =
new SourceCoordinator<>(
+ new JobID(),
OPERATOR_NAME,
new EnumeratorCreatingSource<>(() -> splitEnumerator),
context,
@@ -312,6 +314,7 @@ class SourceCoordinatorTest extends SourceCoordinatorTestBase {
final SourceCoordinator<?, ?> coordinator =
new SourceCoordinator<>(
+ new JobID(),
OPERATOR_NAME,
new EnumeratorCreatingSource<>(
() -> {
@@ -342,6 +345,7 @@ class SourceCoordinatorTest extends SourceCoordinatorTestBase {
};
final SourceCoordinator<?, ?> coordinator =
new SourceCoordinator<>(
+ new JobID(),
OPERATOR_NAME,
new EnumeratorCreatingSource<>(() -> splitEnumerator),
context,
@@ -562,6 +566,7 @@ class SourceCoordinatorTest extends SourceCoordinatorTestBase {
CoordinatorStore store = new CoordinatorStoreImpl();
final SourceCoordinator<?, ?> coordinator =
new SourceCoordinator<>(
+ new JobID(),
OPERATOR_NAME,
createMockSource(),
context,
@@ -583,6 +588,7 @@ class SourceCoordinatorTest extends SourceCoordinatorTestBase {
store.putIfAbsent(listeningID, new SourceEventWrapper(new TestDynamicFilteringEvent()));
final SourceCoordinator<?, ?> coordinator =
new SourceCoordinator<>(
+ new JobID(),
OPERATOR_NAME,
createMockSource(),
context,
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorTestBase.java
index 930923799b1..61ece5c357f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/source/coordinator/SourceCoordinatorTestBase.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.source.coordinator;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.eventtime.WatermarkAlignmentParams;
import org.apache.flink.api.connector.source.Source;
import org.apache.flink.api.connector.source.SourceSplit;
@@ -181,6 +182,7 @@ abstract class SourceCoordinatorTestBase {
createMockSource();
return new SourceCoordinator<>(
+ new JobID(),
OPERATOR_NAME,
mockSource,
getNewSourceCoordinatorContext(),
@@ -201,6 +203,7 @@ abstract class SourceCoordinatorTestBase {
coordinatorThreadName, operatorCoordinatorContext);
SourceCoordinatorContext<MockSourceSplit> coordinatorContext =
new SourceCoordinatorContext<>(
+ new JobID(),
Executors.newScheduledThreadPool(1, coordinatorThreadFactory),
Executors.newScheduledThreadPool(
1, new ExecutorThreadFactory(coordinatorThreadName + "-worker")),
diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/JobIDLoggingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/JobIDLoggingITCase.java
index e2ff4556190..2e8ac6a3ff0 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/misc/JobIDLoggingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/misc/JobIDLoggingITCase.java
@@ -20,7 +20,10 @@ package org.apache.flink.test.misc;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
+import org.apache.flink.api.common.eventtime.AscendingTimestampsWatermarks;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.api.connector.source.lib.NumberSequenceSource;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.JobManagerOptions;
@@ -30,6 +33,7 @@ import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.executiongraph.ExecutionGraph;
import org.apache.flink.runtime.jobmaster.JobMaster;
import org.apache.flink.runtime.scheduler.adaptive.AdaptiveScheduler;
+import org.apache.flink.runtime.source.coordinator.SourceCoordinator;
import org.apache.flink.runtime.taskexecutor.TaskExecutor;
import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
@@ -70,6 +74,10 @@ class JobIDLoggingITCase {
public final LoggerAuditingExtension checkpointCoordinatorLogging =
new LoggerAuditingExtension(CheckpointCoordinator.class, DEBUG);
+ @RegisterExtension
+ public final LoggerAuditingExtension sourceCoordinatorLogging =
+ new LoggerAuditingExtension(SourceCoordinator.class, DEBUG);
+
@RegisterExtension
public final LoggerAuditingExtension streamTaskLogging =
new LoggerAuditingExtension(StreamTask.class, DEBUG);
@@ -136,6 +144,15 @@ class JobIDLoggingITCase {
"Completed checkpoint .*",
"Checkpoint state: .*"));
+ assertJobIDPresent(
+ jobID,
+ sourceCoordinatorLogging,
+ asList(
+ "Starting split enumerator.*",
+ "Distributing maxAllowedWatermark.*",
+ "Source .* registering reader for parallel task.*",
+ "Closing SourceCoordinator for source .*"));
+
assertJobIDPresent(
jobID,
streamTaskLogging,
@@ -272,7 +289,15 @@ class JobIDLoggingITCase {
private static JobID runJob(ClusterClient<?> clusterClient) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.fromSequence(Long.MIN_VALUE, Long.MAX_VALUE).addSink(new DiscardingSink<>());
+
+ env.fromSource(
+ new NumberSequenceSource(0, Long.MAX_VALUE),
+ WatermarkStrategy.forGenerator(ctx -> new AscendingTimestampsWatermarks())
+ .withWatermarkAlignment(
+ "group-1", Duration.ofMillis(1000), Duration.ofMillis(1))
+ .withTimestampAssigner((r, t) -> (long) r),
+ "Source-42441337")
+ .addSink(new DiscardingSink<>());
JobID jobId = clusterClient.submitJob(env.getStreamGraph().getJobGraph()).get();
Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5));
while (deadline.hasTimeLeft()