You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/22 10:05:41 UTC
(seatunnel) 02/07: [Test][Zeta] Add a regression test for the deploy hang in #11679
This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch davidzollo_republish_11727_start_latch_release
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 25a5766ea3f22a298fefab5e8e778a1390085628
Author: SAADAOUI ABDESSALEM <10...@users.noreply.github.com>
AuthorDate: Sun Aug 9 16:18:02 2026 +0200
[Test][Zeta] Add a regression test for the deploy hang in #11679
Asserts the narrow contract that prevents the hang: deploying a task group must
return even if its execution context disappears while the deployment is in
flight. On the unpatched engine it fails after 30s on the first iteration; with
the fix the same 30 iterations complete in under 4s.
(cherry picked from commit 23011fd7d4812a9e28df83fe4ce2430645d77e6e)
---
.../server/TaskDeployStaleContextRaceTest.java | 166 +++++++++++++++++++++
1 file changed, 166 insertions(+)
diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskDeployStaleContextRaceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskDeployStaleContextRaceTest.java
new file mode 100644
index 0000000000..b3b7357fc4
--- /dev/null
+++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskDeployStaleContextRaceTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.engine.server;
+
+import org.apache.seatunnel.engine.server.execution.TaskDeployState;
+import org.apache.seatunnel.engine.server.execution.TaskGroupContext;
+import org.apache.seatunnel.engine.server.execution.TaskGroupLocation;
+import org.apache.seatunnel.engine.server.execution.TaskGroupType;
+import org.apache.seatunnel.engine.server.execution.TestTask;
+import org.apache.seatunnel.engine.server.task.TaskGroupImmutableInformation;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import com.hazelcast.internal.serialization.Data;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static java.util.Collections.emptySet;
+
+/**
+ * Regression test for <a href="https://github.com/apache/seatunnel/issues/11679">#11679</a>.
+ *
+ * <p>{@code TaskGroupLocation} is {jobId, pipelineId, taskGroupId} and is reused verbatim across
+ * pipeline restore generations. {@code TaskGroupExecutionTracker.taskDone()} removes the entry for
+ * that location from {@code executionContexts}, so a late {@code taskDone()} belonging to the
+ * previous generation can delete the context that the current generation's {@code
+ * deployLocalTask()} has just installed.
+ *
+ * <p>When that happens, {@code BlockingWorker.run()} resolves its class loader through {@code
+ * executionContexts.get(location)} <em>before</em> its {@code try} block, and {@code
+ * startedLatch.countDown()} sits <em>inside</em> it. A missing context therefore throws before the
+ * latch is ever counted down, the exception is swallowed by the submitting {@code Future}, and
+ * {@code submitBlockingTask()} waits on {@code startedLatch.await()} forever — while holding the
+ * {@code SubPlan} monitor, which in turn blocks checkpoint-error handling from ever moving the
+ * pipeline to a terminal state.
+ *
+ * <p>This test asserts the narrow contract that prevents the hang: <b>deploying a task group must
+ * return, even if the execution context for that location disappears while the deployment is in
+ * flight.</b> Whether it returns successfully or throws is not asserted — only that it does not
+ * block indefinitely.
+ *
+ * <p>The race window is small, so the test repeats the deployment and races a remover thread
+ * against it. It can therefore fail to <em>detect</em> a regression on an unlucky run, but it
+ * cannot report a failure that is not real: the only way it fails is a deployment that never
+ * returns.
+ */
+public class TaskDeployStaleContextRaceTest extends AbstractSeaTunnelServerTest {
+
+ /** Deployment is a local, in-memory operation; anything beyond this is the hang. */
+ private static final long DEPLOY_TIMEOUT_SECONDS = 30;
+
+ private static final int ITERATIONS = 30;
+
+ @Test
+ public void deployMustNotHangWhenExecutionContextDisappearsDuringDeploy() throws Exception {
+ TaskExecutionService taskExecutionService = server.getTaskExecutionService();
+ ConcurrentMap<TaskGroupLocation, TaskGroupContext> executionContexts =
+ executionContextsOf(taskExecutionService);
+
+ ExecutorService deployer = Executors.newSingleThreadExecutor();
+ try {
+ for (int iteration = 0; iteration < ITERATIONS; iteration++) {
+ long jobId = System.nanoTime();
+ TaskGroupLocation location = new TaskGroupLocation(jobId, 1, 1);
+ AtomicBoolean stopTask = new AtomicBoolean(false);
+
+ // isThreadsShare() == false routes the task through submitBlockingTask(),
+ // which is the path that waits on startedLatch.
+ TestTask blockingTask = new TestTask(stopTask, 300, false);
+
+ TaskGroupImmutableInformation information =
+ new TaskGroupImmutableInformation(
+ jobId,
+ 1,
+ TaskGroupType.INTERMEDIATE_BLOCKING_QUEUE,
+ location,
+ "staleContextRace",
+ Collections.singletonList(
+ nodeEngine.getSerializationService().toData(blockingTask)),
+ Collections.singletonList(emptySet()),
+ Arrays.asList(emptySet()));
+ Data data = nodeEngine.getSerializationService().toData(information);
+
+ // Stand in for the previous generation's late taskDone(): clear the entry for
+ // this location while the deployment is in flight.
+ AtomicBoolean stopRemover = new AtomicBoolean(false);
+ Thread remover =
+ new Thread(
+ () -> {
+ while (!stopRemover.get()) {
+ executionContexts.remove(location);
+ }
+ },
+ "stale-taskDone-simulator");
+ remover.setDaemon(true);
+ remover.start();
+
+ Future<TaskDeployState> deployment =
+ deployer.submit(() -> taskExecutionService.deployTask(data));
+ try {
+ // Returning normally and throwing both satisfy the contract. Only a
+ // deployment that never returns is the defect under test.
+ deployment.get(DEPLOY_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ } catch (ExecutionException expectedWhenContextIsGone) {
+ // deployTask surfaced the missing context as an error - acceptable.
+ } catch (TimeoutException e) {
+ deployment.cancel(true);
+ Assertions.fail(
+ "deployTask did not return within "
+ + DEPLOY_TIMEOUT_SECONDS
+ + "s on iteration "
+ + iteration
+ + ". A BlockingWorker that fails before"
+ + " startedLatch.countDown() leaves submitBlockingTask"
+ + " waiting forever while holding the SubPlan monitor."
+ + " See https://github.com/apache/seatunnel/issues/11679");
+ } finally {
+ stopRemover.set(true);
+ remover.join(TimeUnit.SECONDS.toMillis(5));
+ stopTask.set(true);
+ try {
+ taskExecutionService.cancelTaskGroup(location);
+ } catch (RuntimeException ignored) {
+ // the group may already be gone; irrelevant to this assertion
+ }
+ }
+ }
+ } finally {
+ deployer.shutdownNow();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static ConcurrentMap<TaskGroupLocation, TaskGroupContext> executionContextsOf(
+ TaskExecutionService taskExecutionService) throws Exception {
+ Field field = TaskExecutionService.class.getDeclaredField("executionContexts");
+ field.setAccessible(true);
+ return (ConcurrentMap<TaskGroupLocation, TaskGroupContext>) field.get(taskExecutionService);
+ }
+}