You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by gi...@apache.org on 2026/08/28 03:18:13 UTC
(seatunnel) branch dev updated: [Fix][Zeta] Release classloaders after failed task deployment (#11812)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 1ff72ceb3d [Fix][Zeta] Release classloaders after failed task deployment (#11812)
1ff72ceb3d is described below
commit 1ff72ceb3d2a823a0fca64ec009984d75e846c28
Author: Goutam Adwant <86...@users.noreply.github.com>
AuthorDate: Fri Aug 28 03:08:35 2026 +0000
[Fix][Zeta] Release classloaders after failed task deployment (#11812)
Signed-off-by: Goutam Adwant <wo...@gmail.com>
Signed-off-by: goutamadwant <wo...@gmail.com>
Co-authored-by: DanielLeens <es...@outlook.com>
Co-authored-by: David Zollo <da...@gmail.com>
---
.../engine/server/TaskExecutionService.java | 120 ++++++++++++++++-----
.../engine/server/TaskExecutionServiceTest.java | 119 ++++++++++++++++++++
2 files changed, 210 insertions(+), 29 deletions(-)
diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
index eeef577ad2..7fd2be4c43 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
@@ -104,6 +104,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
import java.util.stream.Stream;
import static com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch;
@@ -447,6 +448,8 @@ public class TaskExecutionService implements DynamicMetricsProvider {
"received deploying task executionId [%s]",
taskImmutableInfo.getExecutionId()));
TaskGroup taskGroup = null;
+ // References owned by this deployment attempt until TaskGroupContext is published.
+ List<Collection<URL>> acquiredClassLoaderJars = new ArrayList<>();
try {
List<Set<ConnectorJarIdentifier>> connectorJarIdentifiersList =
taskImmutableInfo.getConnectorJarIdentifiers();
@@ -469,9 +472,11 @@ public class TaskExecutionService implements DynamicMetricsProvider {
} else if (!CollectionUtils.isEmpty(taskImmutableInfo.getJars().get(i))) {
jars = taskImmutableInfo.getJars().get(i);
}
+ List<URL> classLoaderJars = Lists.newArrayList(jars);
ClassLoader classLoader =
classLoaderService.getClassLoader(
- taskImmutableInfo.getJobId(), Lists.newArrayList(jars));
+ taskImmutableInfo.getJobId(), classLoaderJars);
+ acquiredClassLoaderJars.add(classLoaderJars);
Task task;
if (jars.isEmpty()) {
task = nodeEngine.getSerializationService().toObject(taskData.get(i));
@@ -484,7 +489,7 @@ public class TaskExecutionService implements DynamicMetricsProvider {
}
tasks.add(task);
classLoaders.put(task.getTaskID(), classLoader);
- taskJars.put(task.getTaskID(), jars);
+ taskJars.put(task.getTaskID(), classLoaderJars);
}
taskGroup =
TaskGroupUtils.createTaskGroup(
@@ -516,12 +521,30 @@ public class TaskExecutionService implements DynamicMetricsProvider {
classLoaderService.releaseClassLoader(
taskImmutableInfo.getJobId(), entry.getValue());
}
+ acquiredClassLoaderJars.clear();
return TaskDeployState.success();
}
- deployLocalTask(taskGroup, classLoaders, taskJars);
+ AtomicBoolean classLoaderOwnershipTransferred = new AtomicBoolean();
+ deployLocalTask(
+ taskGroup,
+ classLoaders,
+ taskJars,
+ () -> classLoaderOwnershipTransferred.set(true),
+ failure -> {
+ releaseClassLoadersAfterFailedDeployment(
+ taskImmutableInfo.getJobId(), acquiredClassLoaderJars, failure);
+ acquiredClassLoaderJars.clear();
+ });
+ // Publication is a monotonic ownership transfer. The context may already have
+ // completed and left executionContexts by the time deployment returns.
+ if (classLoaderOwnershipTransferred.get()) {
+ acquiredClassLoaderJars.clear();
+ }
return TaskDeployState.success();
}
} catch (Throwable t) {
+ releaseClassLoadersAfterFailedDeployment(
+ taskImmutableInfo.getJobId(), acquiredClassLoaderJars, t);
logger.severe(
String.format(
"TaskGroupID : %s deploy error with Exception: %s",
@@ -533,6 +556,30 @@ public class TaskExecutionService implements DynamicMetricsProvider {
}
}
+ /**
+ * Releases classloader references acquired by the current deployment attempt.
+ *
+ * <p>This cleanup only releases references that have not been transferred to a published {@link
+ * TaskGroupContext}. Cleanup failures are added to the deployment failure so all references are
+ * attempted without replacing the original error.
+ */
+ private void releaseClassLoadersAfterFailedDeployment(
+ long jobId,
+ List<Collection<URL>> acquiredClassLoaderJars,
+ Throwable deploymentFailure) {
+ // Release in reverse acquisition order, matching the ownership stack built above.
+ for (int i = acquiredClassLoaderJars.size() - 1; i >= 0; i--) {
+ Collection<URL> jars = acquiredClassLoaderJars.get(i);
+ try {
+ classLoaderService.releaseClassLoader(jobId, jars);
+ } catch (Throwable cleanupFailure) {
+ deploymentFailure.addSuppressed(cleanupFailure);
+ logger.severe(
+ "Release classloader after failed task deployment failed", cleanupFailure);
+ }
+ }
+ }
+
/**
* Deploys a task group locally. This method initializes the task group, creates execution
* contexts, and submits tasks for execution based on the configured thread share mode.
@@ -546,7 +593,42 @@ public class TaskExecutionService implements DynamicMetricsProvider {
@NonNull TaskGroup taskGroup,
@NonNull ConcurrentHashMap<Long, ClassLoader> classLoaders,
ConcurrentHashMap<Long, Collection<URL>> jars) {
+ return deployLocalTask(taskGroup, classLoaders, jars, () -> {}, failure -> {});
+ }
+
+ private PassiveCompletableFuture<TaskExecutionState> deployLocalTask(
+ @NonNull TaskGroup taskGroup,
+ @NonNull ConcurrentHashMap<Long, ClassLoader> classLoaders,
+ ConcurrentHashMap<Long, Collection<URL>> jars,
+ Runnable onContextPublished,
+ Consumer<Throwable> onFailureBeforeContextPublished) {
CompletableFuture<TaskExecutionState> resultFuture = new CompletableFuture<>();
+ resultFuture.whenCompleteAsync(
+ withTryCatch(
+ logger,
+ (r, s) -> {
+ if (s != null) {
+ logger.severe(
+ String.format(
+ "Task %s complete with error %s",
+ taskGroup.getTaskGroupLocation(),
+ ExceptionUtils.getMessage(s)));
+ }
+ if (r == null) {
+ r =
+ new TaskExecutionState(
+ taskGroup.getTaskGroupLocation(),
+ ExecutionState.FAILED,
+ s);
+ }
+ logger.info(
+ String.format(
+ "Task %s complete with state %s",
+ r.getTaskGroupLocation(), r.getExecutionState()));
+ notifyTaskStatusToMaster(taskGroup.getTaskGroupLocation(), r);
+ }),
+ MDCTracer.tracing(executorService));
+ boolean contextPublished = false;
try {
taskGroup.init();
logger.info(
@@ -591,6 +673,8 @@ public class TaskExecutionService implements DynamicMetricsProvider {
executionContexts.put(
taskGroup.getTaskGroupLocation(),
new TaskGroupContext(taskGroup, classLoaders, jars));
+ contextPublished = true;
+ onContextPublished.run();
cancellationFutures.put(taskGroup.getTaskGroupLocation(), cancellationFuture);
submitThreadShareTask(executionTracker, byCooperation.get(true));
submitBlockingTask(executionTracker, byCooperation.get(false));
@@ -600,33 +684,11 @@ public class TaskExecutionService implements DynamicMetricsProvider {
"deploying TaskGroup %s success", taskGroup.getTaskGroupLocation()));
} catch (Throwable t) {
logger.severe(ExceptionUtils.getMessage(t));
+ if (!contextPublished) {
+ onFailureBeforeContextPublished.accept(t);
+ }
resultFuture.completeExceptionally(t);
}
- resultFuture.whenCompleteAsync(
- withTryCatch(
- logger,
- (r, s) -> {
- if (s != null) {
- logger.severe(
- String.format(
- "Task %s complete with error %s",
- taskGroup.getTaskGroupLocation(),
- ExceptionUtils.getMessage(s)));
- }
- if (r == null) {
- r =
- new TaskExecutionState(
- taskGroup.getTaskGroupLocation(),
- ExecutionState.FAILED,
- s);
- }
- logger.info(
- String.format(
- "Task %s complete with state %s",
- r.getTaskGroupLocation(), r.getExecutionState()));
- notifyTaskStatusToMaster(taskGroup.getTaskGroupLocation(), r);
- }),
- MDCTracer.tracing(executorService));
return new PassiveCompletableFuture<>(resultFuture);
}
@@ -637,7 +699,7 @@ public class TaskExecutionService implements DynamicMetricsProvider {
* @param taskGroupLocation the location of the task group
* @param taskExecutionState the execution state to report
*/
- private void notifyTaskStatusToMaster(
+ void notifyTaskStatusToMaster(
TaskGroupLocation taskGroupLocation, TaskExecutionState taskExecutionState) {
long sleepTime = 1000;
boolean notifyStateSuccess = false;
diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
index 3b6649d296..4c98fd977b 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
@@ -21,12 +21,16 @@ import org.apache.seatunnel.shade.com.google.common.collect.Lists;
import org.apache.seatunnel.engine.common.utils.PassiveCompletableFuture;
import org.apache.seatunnel.engine.common.utils.concurrent.CompletableFuture;
+import org.apache.seatunnel.engine.core.classloader.DefaultClassLoaderService;
+import org.apache.seatunnel.engine.server.exception.TaskGroupContextNotFoundException;
import org.apache.seatunnel.engine.server.execution.BlockTask;
import org.apache.seatunnel.engine.server.execution.ExceptionTestTask;
import org.apache.seatunnel.engine.server.execution.FixedCallTestTimeTask;
+import org.apache.seatunnel.engine.server.execution.ProgressState;
import org.apache.seatunnel.engine.server.execution.StopTimeTestTask;
import org.apache.seatunnel.engine.server.execution.Task;
import org.apache.seatunnel.engine.server.execution.TaskDeployState;
+import org.apache.seatunnel.engine.server.execution.TaskExecutionContext;
import org.apache.seatunnel.engine.server.execution.TaskExecutionState;
import org.apache.seatunnel.engine.server.execution.TaskGroup;
import org.apache.seatunnel.engine.server.execution.TaskGroupContext;
@@ -41,6 +45,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
import com.hazelcast.flakeidgen.FlakeIdGenerator;
import com.hazelcast.internal.serialization.Data;
@@ -54,6 +59,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledFuture;
@@ -234,6 +240,101 @@ public class TaskExecutionServiceTest extends AbstractSeaTunnelServerTest {
console.delete();
}
+ /**
+ * Verifies that a partially constructed task group does not retain classloader references when
+ * a later task cannot be deserialized.
+ */
+ @Test
+ public void testDeployTaskReleasesClassLoadersWhenDeserializationFails() throws IOException {
+ TaskExecutionService taskExecutionService = server.getTaskExecutionService();
+ DefaultClassLoaderService classLoaderService =
+ (DefaultClassLoaderService) server.getClassLoaderService();
+
+ File testJar = File.createTempFile("failed-deployment", ".jar");
+ testJar.deleteOnExit();
+ URL testJarUrl = testJar.toURI().toURL();
+ Set<URL> testJars = Collections.singleton(testJarUrl);
+ long testJobId = System.currentTimeMillis();
+ TestTask validTask = new TestTask(new AtomicBoolean(false), 300, true);
+ TaskGroupImmutableInformation taskGroupImmutableInformation =
+ new TaskGroupImmutableInformation(
+ testJobId,
+ 1,
+ TaskGroupType.INTERMEDIATE_BLOCKING_QUEUE,
+ new TaskGroupLocation(testJobId, 1, 1),
+ "testDeployTaskReleasesClassLoadersWhenDeserializationFails",
+ Arrays.asList(
+ nodeEngine.getSerializationService().toData(validTask),
+ nodeEngine.getSerializationService().toData("not a task")),
+ Arrays.asList(testJars, testJars),
+ Arrays.asList(emptySet(), emptySet()));
+
+ TaskDeployState taskDeployState =
+ taskExecutionService.deployTask(taskGroupImmutableInformation);
+
+ Assertions.assertFalse(taskDeployState.isSuccess());
+ Assertions.assertThrows(
+ TaskGroupContextNotFoundException.class,
+ () ->
+ taskExecutionService.getActiveExecutionContext(
+ taskGroupImmutableInformation.getTaskGroupLocation()));
+ Assertions.assertTrue(
+ classLoaderService.queryClassLoaderById(testJobId, testJars).isPresent());
+ Assertions.assertEquals(
+ 0, classLoaderService.queryClassLoaderReferenceCount(testJobId, testJars));
+ testJar.delete();
+ }
+
+ /**
+ * Verifies that a failure before context publication releases the acquired classloader and is
+ * still reported to the master.
+ */
+ @Test
+ public void testDeployTaskHandlesFailureBeforeContextPublication() throws IOException {
+ TaskExecutionService taskExecutionService = Mockito.spy(server.getTaskExecutionService());
+ Mockito.doNothing()
+ .when(taskExecutionService)
+ .notifyTaskStatusToMaster(Mockito.any(), Mockito.any());
+ DefaultClassLoaderService classLoaderService =
+ (DefaultClassLoaderService) server.getClassLoaderService();
+
+ File testJar = File.createTempFile("failed-context-publication", ".jar");
+ testJar.deleteOnExit();
+ URL testJarUrl = testJar.toURI().toURL();
+ Set<URL> testJars = Collections.singleton(testJarUrl);
+ long testJobId = System.currentTimeMillis();
+ TaskGroupLocation location = new TaskGroupLocation(testJobId, 1, 1);
+ Task task = new ContextInitializationFailureTask();
+ TaskGroupImmutableInformation taskGroupImmutableInformation =
+ new TaskGroupImmutableInformation(
+ testJobId,
+ 1,
+ TaskGroupType.DEFAULT,
+ location,
+ "testDeployTaskHandlesFailureBeforeContextPublication",
+ Collections.singletonList(
+ nodeEngine.getSerializationService().toData(task)),
+ Collections.singletonList(testJars),
+ Collections.singletonList(emptySet()));
+
+ TaskDeployState taskDeployState =
+ taskExecutionService.deployTask(taskGroupImmutableInformation);
+
+ Assertions.assertEquals(TaskDeployState.success(), taskDeployState);
+ Assertions.assertThrows(
+ TaskGroupContextNotFoundException.class,
+ () -> taskExecutionService.getActiveExecutionContext(location));
+ Assertions.assertTrue(
+ classLoaderService.queryClassLoaderById(testJobId, testJars).isPresent());
+ Assertions.assertEquals(
+ 0, classLoaderService.queryClassLoaderReferenceCount(testJobId, testJars));
+ Mockito.verify(taskExecutionService, Mockito.timeout(5000))
+ .notifyTaskStatusToMaster(
+ Mockito.eq(location),
+ Mockito.argThat(state -> state.getExecutionState() == FAILED));
+ testJar.delete();
+ }
+
/** Test task execution time is the same as the timer timeout */
@Test
public void testCriticalCallTime() throws InterruptedException {
@@ -539,4 +640,22 @@ public class TaskExecutionServiceTest extends AbstractSeaTunnelServerTest {
Assertions.assertDoesNotThrow(() -> taskExecutionService.closeTimerFlushTask(unknown));
}
+
+ private static class ContextInitializationFailureTask implements Task {
+
+ @Override
+ public void setTaskExecutionContext(TaskExecutionContext taskExecutionContext) {
+ throw new IllegalStateException("context initialization failed");
+ }
+
+ @Override
+ public ProgressState call() {
+ return ProgressState.DONE;
+ }
+
+ @Override
+ public Long getTaskID() {
+ return 1L;
+ }
+ }
}