You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/22 10:05:40 UTC
(seatunnel) 01/07: [Fix][Zeta] Release the start latch when a blocking worker cannot start
This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch davidzollo_republish_11727_start_latch_release
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 4cbe31d1649d6f73a6634f08d244d7c3d4075663
Author: SAADAOUI ABDESSALEM <10...@users.noreply.github.com>
AuthorDate: Sun Aug 9 16:18:01 2026 +0200
[Fix][Zeta] Release the start latch when a blocking worker cannot start
BlockingWorker.run() resolved its class loader from executionContexts before its
try block, while startedLatch.countDown() sat inside it. TaskGroupLocation is
reused verbatim across pipeline restore generations, so a taskDone() from an
earlier generation can remove the context the current deployLocalTask() has just
installed. The worker then threw before signalling, the exception was swallowed
by the submitting Future, and submitBlockingTask() waited on the latch forever
while holding the SubPlan monitor - which prevented checkpoint error handling
from moving the pipeline to a terminal state and left the seatunnel.sh client
blocked indefinitely.
Move the lookup inside the try, fail explicitly when the context is gone, and
release the latch exactly once per worker on that path. The signal still happens
before init(), so submitBlockingTask() still returns once workers have started
rather than once they have finished.
(cherry picked from commit b91e00582827218ca72013165f6d4c9a2800e0ab)
---
.../engine/server/TaskExecutionService.java | 37 ++++++++++++++++++----
1 file changed, 31 insertions(+), 6 deletions(-)
diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
index eeef577ad2..c02d3fdea5 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
@@ -1067,17 +1067,36 @@ public class TaskExecutionService implements DynamicMetricsProvider {
public void run() {
TaskExecutionService.TaskGroupExecutionTracker taskGroupExecutionTracker =
tracker.taskGroupExecutionTracker;
- ClassLoader classLoader =
- executionContexts
- .get(taskGroupExecutionTracker.taskGroup.getTaskGroupLocation())
- .getClassLoaders()
- .get(tracker.task.getTaskID());
+ TaskGroupLocation taskGroupLocation =
+ taskGroupExecutionTracker.taskGroup.getTaskGroupLocation();
ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader();
- Thread.currentThread().setContextClassLoader(classLoader);
final Task t = tracker.task;
ProgressState result = null;
+ boolean startSignalled = false;
try {
+ // Resolving the class loader has to happen inside this try. The execution
+ // context for this location can already have been removed by a taskDone() of
+ // an earlier restore generation - TaskGroupLocation is reused verbatim across
+ // restores - and dereferencing a missing context threw before startedLatch was
+ // ever counted down. submitBlockingTask() then waited on that latch forever
+ // while holding the SubPlan monitor, which stopped the pipeline from reaching a
+ // terminal state and left the submitting client blocked. See #11679.
+ TaskGroupContext taskGroupContext = executionContexts.get(taskGroupLocation);
+ if (taskGroupContext == null) {
+ throw new IllegalStateException(
+ String.format(
+ "Execution context for %s is no longer registered; the task"
+ + " group was cleaned up while it was being deployed",
+ taskGroupLocation));
+ }
+ Thread.currentThread()
+ .setContextClassLoader(
+ taskGroupContext.getClassLoaders().get(t.getTaskID()));
+
+ // Signalled before init() so that submitBlockingTask() returns once workers
+ // have started, not once they have finished.
startedLatch.countDown();
+ startSignalled = true;
t.init();
do {
result = t.call();
@@ -1098,6 +1117,12 @@ public class TaskExecutionService implements DynamicMetricsProvider {
}
taskGroupExecutionTracker.exception(e);
} finally {
+ // A worker that failed before signalling must still release the deployer,
+ // otherwise submitBlockingTask() blocks forever. Guarded so each worker counts
+ // down exactly once.
+ if (!startSignalled) {
+ startedLatch.countDown();
+ }
taskGroupExecutionTracker.taskDone(t);
if (result == null || !result.isDone()) {
try {