You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/22 10:05:45 UTC

(seatunnel) 06/07: [Test][Zeta] Assert the task group future settles on the stale-context path

This is an automated email from the ASF dual-hosted git repository.

davidzollo pushed a commit to branch davidzollo_republish_11727_start_latch_release
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit 85bdfc189f3041cc74e9c4a4be9272cb8d7c2a53
Author: SAADAOUI ABDESSALEM <10...@users.noreply.github.com>
AuthorDate: Mon Aug 10 14:58:57 2026 +0200

    [Test][Zeta] Assert the task group future settles on the stale-context path
    
    deployTask(Data) discards the future returned by deployLocalTask(), so the
    existing test can only observe that the deployment returned - which is already
    satisfied when startedLatch is released, before taskDone() runs. A failure
    inside taskDone() therefore leaves the task group's TaskExecutionState future
    pending forever while that test still passes.
    
    That future is what PhysicalVertex and SubPlan wait on to drive pipeline state,
    so never completing it reproduces a hang of the same class as #11679 one level
    up. The new test calls deployLocalTask() directly to hold the future and keeps
    the remover racing until after it has been observed. Completing exceptionally
    satisfies the contract; only never completing is the defect.
    
    Verified on the JDK 8 leg against the same commit, test unchanged in both:
    with the taskDone guard reverted the new method fails on iteration 0 while the
    existing method still passes; with the guard restored both pass in 4.5s.
    
    Raised as Issue 3 in review.
    
    (cherry picked from commit e9b25d84d55f6b0afcf27444c12858ef3a93cdaa)
---
 .../server/TaskDeployStaleContextRaceTest.java     | 111 ++++++++++++++++++++-
 1 file changed, 109 insertions(+), 2 deletions(-)

diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskDeployStaleContextRaceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskDeployStaleContextRaceTest.java
index 22b808dee6..4289aed6c6 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskDeployStaleContextRaceTest.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskDeployStaleContextRaceTest.java
@@ -17,8 +17,12 @@
 
 package org.apache.seatunnel.engine.server;
 
+import org.apache.seatunnel.engine.common.utils.PassiveCompletableFuture;
+import org.apache.seatunnel.engine.server.execution.Task;
 import org.apache.seatunnel.engine.server.execution.TaskDeployState;
+import org.apache.seatunnel.engine.server.execution.TaskExecutionState;
 import org.apache.seatunnel.engine.server.execution.TaskGroupContext;
+import org.apache.seatunnel.engine.server.execution.TaskGroupDefaultImpl;
 import org.apache.seatunnel.engine.server.execution.TaskGroupLocation;
 import org.apache.seatunnel.engine.server.execution.TaskGroupType;
 import org.apache.seatunnel.engine.server.execution.TestTask;
@@ -30,8 +34,11 @@ import org.junit.jupiter.api.Test;
 import com.hazelcast.internal.serialization.Data;
 
 import java.lang.reflect.Field;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -56,13 +63,13 @@ import static java.util.Collections.emptySet;
  * executionContexts.get(location)} <em>before</em> its {@code try} block, and {@code
  * startedLatch.countDown()} sits <em>inside</em> it. A missing context therefore throws before the
  * latch is ever counted down, the exception is swallowed by the submitting {@code Future}, and
- * {@code submitBlockingTask()} waits on {@code startedLatch.await()} forever — while holding the
+ * {@code submitBlockingTask()} waits on {@code startedLatch.await()} forever - while holding the
  * {@code SubPlan} monitor, which in turn blocks checkpoint-error handling from ever moving the
  * pipeline to a terminal state.
  *
  * <p>This test asserts the narrow contract that prevents the hang: <b>deploying a task group must
  * return, even if the execution context for that location disappears while the deployment is in
- * flight.</b> Whether it returns successfully or throws is not asserted — only that it does not
+ * flight.</b> Whether it returns successfully or throws is not asserted - only that it does not
  * block indefinitely.
  *
  * <p>The race window is small, so the test repeats the deployment and races a remover thread
@@ -160,6 +167,106 @@ public class TaskDeployStaleContextRaceTest extends AbstractSeaTunnelServerTest
         }
     }
 
+    /**
+     * The sibling contract to the test above, on the other side of the same race.
+     *
+     * <p>{@code deployTask(Data)} discards the future returned by {@code deployLocalTask()}, so a
+     * test driven through it can only observe that the deployment returned - which is satisfied the
+     * moment {@code startedLatch} is released, before {@code taskDone()} runs. A failure inside
+     * {@code taskDone()} therefore leaves the task group's completion future pending forever while
+     * that test still passes.
+     *
+     * <p>That future is the one {@code PhysicalVertex} and {@code SubPlan} wait on to drive
+     * pipeline state, so never completing it reproduces a hang of the same class as the one this
+     * class is named for, one level up. This test calls {@code deployLocalTask()} directly to hold
+     * the future, and keeps the remover racing until after it has been observed.
+     *
+     * <p>Completing exceptionally satisfies the contract; only never completing is the defect.
+     */
+    @Test
+    public void taskGroupFutureMustCompleteWhenExecutionContextDisappearsDuringDeploy()
+            throws Exception {
+        TaskExecutionService taskExecutionService = server.getTaskExecutionService();
+        ConcurrentMap<TaskGroupLocation, TaskGroupContext> executionContexts =
+                executionContextsOf(taskExecutionService);
+
+        ExecutorService deployer = Executors.newSingleThreadExecutor();
+        try {
+            for (int iteration = 0; iteration < ITERATIONS; iteration++) {
+                long jobId = System.nanoTime();
+                TaskGroupLocation location = new TaskGroupLocation(jobId, 1, 1);
+                AtomicBoolean stopTask = new AtomicBoolean(false);
+
+                TestTask blockingTask = new TestTask(stopTask, 300, false);
+                List<Task> tasks = new ArrayList<>();
+                tasks.add(blockingTask);
+                TaskGroupDefaultImpl taskGroup =
+                        new TaskGroupDefaultImpl(location, "staleContextRaceFuture", tasks);
+
+                ConcurrentHashMap<Long, ClassLoader> classLoaders = new ConcurrentHashMap<>();
+                classLoaders.put(
+                        blockingTask.getTaskID(), Thread.currentThread().getContextClassLoader());
+
+                AtomicBoolean stopRemover = new AtomicBoolean(false);
+                Thread remover =
+                        new Thread(
+                                () -> {
+                                    while (!stopRemover.get()) {
+                                        executionContexts.remove(location);
+                                    }
+                                },
+                                "stale-taskDone-simulator");
+                remover.setDaemon(true);
+                remover.start();
+
+                try {
+                    // Deployment itself is covered by the test above; it runs on a separate
+                    // thread here only so a regression there cannot hang this one too.
+                    Future<PassiveCompletableFuture<TaskExecutionState>> deployment =
+                            deployer.submit(
+                                    () ->
+                                            taskExecutionService.deployLocalTask(
+                                                    taskGroup,
+                                                    classLoaders,
+                                                    new ConcurrentHashMap<>()));
+                    PassiveCompletableFuture<TaskExecutionState> completion =
+                            deployment.get(DEPLOY_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+
+                    // Let the task finish on the iterations where the context survived, so
+                    // the group completes for the same reason in both branches.
+                    stopTask.set(true);
+
+                    try {
+                        completion.get(DEPLOY_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+                    } catch (ExecutionException completedExceptionally) {
+                        // Still completed. Only a future that never settles is the defect.
+                    } catch (TimeoutException e) {
+                        Assertions.fail(
+                                "The task group future did not complete within "
+                                        + DEPLOY_TIMEOUT_SECONDS
+                                        + "s on iteration "
+                                        + iteration
+                                        + ". A failure inside taskDone() before"
+                                        + " future.complete() leaves PhysicalVertex and SubPlan"
+                                        + " waiting on a future that never settles."
+                                        + " See https://github.com/apache/seatunnel/issues/11679");
+                    }
+                } finally {
+                    stopRemover.set(true);
+                    remover.join(TimeUnit.SECONDS.toMillis(5));
+                    stopTask.set(true);
+                    try {
+                        taskExecutionService.cancelTaskGroup(location);
+                    } catch (RuntimeException ignored) {
+                        // the group may already be gone; irrelevant to this assertion
+                    }
+                }
+            }
+        } finally {
+            deployer.shutdownNow();
+        }
+    }
+
     @SuppressWarnings("unchecked")
     private static ConcurrentMap<TaskGroupLocation, TaskGroupContext> executionContextsOf(
             TaskExecutionService taskExecutionService) throws Exception {