You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by gi...@apache.org on 2026/08/28 03:08:58 UTC
(seatunnel) branch gh-readonly-queue/dev/pr-11812-a38f63109caf84ef3188003142b3ff98e83e3062 created (now 1ff72ceb3d)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/dev/pr-11812-a38f63109caf84ef3188003142b3ff98e83e3062
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
at 1ff72ceb3d [Fix][Zeta] Release classloaders after failed task deployment (#11812)
This branch includes the following new commits:
new 1ff72ceb3d [Fix][Zeta] Release classloaders after failed task deployment (#11812)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(seatunnel) 01/01: [Fix][Zeta] Release classloaders after failed task deployment (#11812)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-11812-a38f63109caf84ef3188003142b3ff98e83e3062
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 1ff72ceb3d2a823a0fca64ec009984d75e846c28
Author: Goutam Adwant <86...@users.noreply.github.com>
AuthorDate: Fri Aug 28 03:08:35 2026 +0000
[Fix][Zeta] Release classloaders after failed task deployment (#11812)
Signed-off-by: Goutam Adwant <wo...@gmail.com>
Signed-off-by: goutamadwant <wo...@gmail.com>
Co-authored-by: DanielLeens <es...@outlook.com>
Co-authored-by: David Zollo <da...@gmail.com>
---
.../engine/server/TaskExecutionService.java | 120 ++++++++++++++++-----
.../engine/server/TaskExecutionServiceTest.java | 119 ++++++++++++++++++++
2 files changed, 210 insertions(+), 29 deletions(-)
diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
index eeef577ad2..7fd2be4c43 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
@@ -104,6 +104,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
import java.util.stream.Stream;
import static com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch;
@@ -447,6 +448,8 @@ public class TaskExecutionService implements DynamicMetricsProvider {
"received deploying task executionId [%s]",
taskImmutableInfo.getExecutionId()));
TaskGroup taskGroup = null;
+ // References owned by this deployment attempt until TaskGroupContext is published.
+ List<Collection<URL>> acquiredClassLoaderJars = new ArrayList<>();
try {
List<Set<ConnectorJarIdentifier>> connectorJarIdentifiersList =
taskImmutableInfo.getConnectorJarIdentifiers();
@@ -469,9 +472,11 @@ public class TaskExecutionService implements DynamicMetricsProvider {
} else if (!CollectionUtils.isEmpty(taskImmutableInfo.getJars().get(i))) {
jars = taskImmutableInfo.getJars().get(i);
}
+ List<URL> classLoaderJars = Lists.newArrayList(jars);
ClassLoader classLoader =
classLoaderService.getClassLoader(
- taskImmutableInfo.getJobId(), Lists.newArrayList(jars));
+ taskImmutableInfo.getJobId(), classLoaderJars);
+ acquiredClassLoaderJars.add(classLoaderJars);
Task task;
if (jars.isEmpty()) {
task = nodeEngine.getSerializationService().toObject(taskData.get(i));
@@ -484,7 +489,7 @@ public class TaskExecutionService implements DynamicMetricsProvider {
}
tasks.add(task);
classLoaders.put(task.getTaskID(), classLoader);
- taskJars.put(task.getTaskID(), jars);
+ taskJars.put(task.getTaskID(), classLoaderJars);
}
taskGroup =
TaskGroupUtils.createTaskGroup(
@@ -516,12 +521,30 @@ public class TaskExecutionService implements DynamicMetricsProvider {
classLoaderService.releaseClassLoader(
taskImmutableInfo.getJobId(), entry.getValue());
}
+ acquiredClassLoaderJars.clear();
return TaskDeployState.success();
}
- deployLocalTask(taskGroup, classLoaders, taskJars);
+ AtomicBoolean classLoaderOwnershipTransferred = new AtomicBoolean();
+ deployLocalTask(
+ taskGroup,
+ classLoaders,
+ taskJars,
+ () -> classLoaderOwnershipTransferred.set(true),
+ failure -> {
+ releaseClassLoadersAfterFailedDeployment(
+ taskImmutableInfo.getJobId(), acquiredClassLoaderJars, failure);
+ acquiredClassLoaderJars.clear();
+ });
+ // Publication is a monotonic ownership transfer. The context may already have
+ // completed and left executionContexts by the time deployment returns.
+ if (classLoaderOwnershipTransferred.get()) {
+ acquiredClassLoaderJars.clear();
+ }
return TaskDeployState.success();
}
} catch (Throwable t) {
+ releaseClassLoadersAfterFailedDeployment(
+ taskImmutableInfo.getJobId(), acquiredClassLoaderJars, t);
logger.severe(
String.format(
"TaskGroupID : %s deploy error with Exception: %s",
@@ -533,6 +556,30 @@ public class TaskExecutionService implements DynamicMetricsProvider {
}
}
+ /**
+ * Releases classloader references acquired by the current deployment attempt.
+ *
+ * <p>This cleanup only releases references that have not been transferred to a published {@link
+ * TaskGroupContext}. Cleanup failures are added to the deployment failure so all references are
+ * attempted without replacing the original error.
+ */
+ private void releaseClassLoadersAfterFailedDeployment(
+ long jobId,
+ List<Collection<URL>> acquiredClassLoaderJars,
+ Throwable deploymentFailure) {
+ // Release in reverse acquisition order, matching the ownership stack built above.
+ for (int i = acquiredClassLoaderJars.size() - 1; i >= 0; i--) {
+ Collection<URL> jars = acquiredClassLoaderJars.get(i);
+ try {
+ classLoaderService.releaseClassLoader(jobId, jars);
+ } catch (Throwable cleanupFailure) {
+ deploymentFailure.addSuppressed(cleanupFailure);
+ logger.severe(
+ "Release classloader after failed task deployment failed", cleanupFailure);
+ }
+ }
+ }
+
/**
* Deploys a task group locally. This method initializes the task group, creates execution
* contexts, and submits tasks for execution based on the configured thread share mode.
@@ -546,7 +593,42 @@ public class TaskExecutionService implements DynamicMetricsProvider {
@NonNull TaskGroup taskGroup,
@NonNull ConcurrentHashMap<Long, ClassLoader> classLoaders,
ConcurrentHashMap<Long, Collection<URL>> jars) {
+ return deployLocalTask(taskGroup, classLoaders, jars, () -> {}, failure -> {});
+ }
+
+ private PassiveCompletableFuture<TaskExecutionState> deployLocalTask(
+ @NonNull TaskGroup taskGroup,
+ @NonNull ConcurrentHashMap<Long, ClassLoader> classLoaders,
+ ConcurrentHashMap<Long, Collection<URL>> jars,
+ Runnable onContextPublished,
+ Consumer<Throwable> onFailureBeforeContextPublished) {
CompletableFuture<TaskExecutionState> resultFuture = new CompletableFuture<>();
+ resultFuture.whenCompleteAsync(
+ withTryCatch(
+ logger,
+ (r, s) -> {
+ if (s != null) {
+ logger.severe(
+ String.format(
+ "Task %s complete with error %s",
+ taskGroup.getTaskGroupLocation(),
+ ExceptionUtils.getMessage(s)));
+ }
+ if (r == null) {
+ r =
+ new TaskExecutionState(
+ taskGroup.getTaskGroupLocation(),
+ ExecutionState.FAILED,
+ s);
+ }
+ logger.info(
+ String.format(
+ "Task %s complete with state %s",
+ r.getTaskGroupLocation(), r.getExecutionState()));
+ notifyTaskStatusToMaster(taskGroup.getTaskGroupLocation(), r);
+ }),
+ MDCTracer.tracing(executorService));
+ boolean contextPublished = false;
try {
taskGroup.init();
logger.info(
@@ -591,6 +673,8 @@ public class TaskExecutionService implements DynamicMetricsProvider {
executionContexts.put(
taskGroup.getTaskGroupLocation(),
new TaskGroupContext(taskGroup, classLoaders, jars));
+ contextPublished = true;
+ onContextPublished.run();
cancellationFutures.put(taskGroup.getTaskGroupLocation(), cancellationFuture);
submitThreadShareTask(executionTracker, byCooperation.get(true));
submitBlockingTask(executionTracker, byCooperation.get(false));
@@ -600,33 +684,11 @@ public class TaskExecutionService implements DynamicMetricsProvider {
"deploying TaskGroup %s success", taskGroup.getTaskGroupLocation()));
} catch (Throwable t) {
logger.severe(ExceptionUtils.getMessage(t));
+ if (!contextPublished) {
+ onFailureBeforeContextPublished.accept(t);
+ }
resultFuture.completeExceptionally(t);
}
- resultFuture.whenCompleteAsync(
- withTryCatch(
- logger,
- (r, s) -> {
- if (s != null) {
- logger.severe(
- String.format(
- "Task %s complete with error %s",
- taskGroup.getTaskGroupLocation(),
- ExceptionUtils.getMessage(s)));
- }
- if (r == null) {
- r =
- new TaskExecutionState(
- taskGroup.getTaskGroupLocation(),
- ExecutionState.FAILED,
- s);
- }
- logger.info(
- String.format(
- "Task %s complete with state %s",
- r.getTaskGroupLocation(), r.getExecutionState()));
- notifyTaskStatusToMaster(taskGroup.getTaskGroupLocation(), r);
- }),
- MDCTracer.tracing(executorService));
return new PassiveCompletableFuture<>(resultFuture);
}
@@ -637,7 +699,7 @@ public class TaskExecutionService implements DynamicMetricsProvider {
* @param taskGroupLocation the location of the task group
* @param taskExecutionState the execution state to report
*/
- private void notifyTaskStatusToMaster(
+ void notifyTaskStatusToMaster(
TaskGroupLocation taskGroupLocation, TaskExecutionState taskExecutionState) {
long sleepTime = 1000;
boolean notifyStateSuccess = false;
diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
index 3b6649d296..4c98fd977b 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
@@ -21,12 +21,16 @@ import org.apache.seatunnel.shade.com.google.common.collect.Lists;
import org.apache.seatunnel.engine.common.utils.PassiveCompletableFuture;
import org.apache.seatunnel.engine.common.utils.concurrent.CompletableFuture;
+import org.apache.seatunnel.engine.core.classloader.DefaultClassLoaderService;
+import org.apache.seatunnel.engine.server.exception.TaskGroupContextNotFoundException;
import org.apache.seatunnel.engine.server.execution.BlockTask;
import org.apache.seatunnel.engine.server.execution.ExceptionTestTask;
import org.apache.seatunnel.engine.server.execution.FixedCallTestTimeTask;
+import org.apache.seatunnel.engine.server.execution.ProgressState;
import org.apache.seatunnel.engine.server.execution.StopTimeTestTask;
import org.apache.seatunnel.engine.server.execution.Task;
import org.apache.seatunnel.engine.server.execution.TaskDeployState;
+import org.apache.seatunnel.engine.server.execution.TaskExecutionContext;
import org.apache.seatunnel.engine.server.execution.TaskExecutionState;
import org.apache.seatunnel.engine.server.execution.TaskGroup;
import org.apache.seatunnel.engine.server.execution.TaskGroupContext;
@@ -41,6 +45,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
import com.hazelcast.flakeidgen.FlakeIdGenerator;
import com.hazelcast.internal.serialization.Data;
@@ -54,6 +59,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ScheduledFuture;
@@ -234,6 +240,101 @@ public class TaskExecutionServiceTest extends AbstractSeaTunnelServerTest {
console.delete();
}
+ /**
+ * Verifies that a partially constructed task group does not retain classloader references when
+ * a later task cannot be deserialized.
+ */
+ @Test
+ public void testDeployTaskReleasesClassLoadersWhenDeserializationFails() throws IOException {
+ TaskExecutionService taskExecutionService = server.getTaskExecutionService();
+ DefaultClassLoaderService classLoaderService =
+ (DefaultClassLoaderService) server.getClassLoaderService();
+
+ File testJar = File.createTempFile("failed-deployment", ".jar");
+ testJar.deleteOnExit();
+ URL testJarUrl = testJar.toURI().toURL();
+ Set<URL> testJars = Collections.singleton(testJarUrl);
+ long testJobId = System.currentTimeMillis();
+ TestTask validTask = new TestTask(new AtomicBoolean(false), 300, true);
+ TaskGroupImmutableInformation taskGroupImmutableInformation =
+ new TaskGroupImmutableInformation(
+ testJobId,
+ 1,
+ TaskGroupType.INTERMEDIATE_BLOCKING_QUEUE,
+ new TaskGroupLocation(testJobId, 1, 1),
+ "testDeployTaskReleasesClassLoadersWhenDeserializationFails",
+ Arrays.asList(
+ nodeEngine.getSerializationService().toData(validTask),
+ nodeEngine.getSerializationService().toData("not a task")),
+ Arrays.asList(testJars, testJars),
+ Arrays.asList(emptySet(), emptySet()));
+
+ TaskDeployState taskDeployState =
+ taskExecutionService.deployTask(taskGroupImmutableInformation);
+
+ Assertions.assertFalse(taskDeployState.isSuccess());
+ Assertions.assertThrows(
+ TaskGroupContextNotFoundException.class,
+ () ->
+ taskExecutionService.getActiveExecutionContext(
+ taskGroupImmutableInformation.getTaskGroupLocation()));
+ Assertions.assertTrue(
+ classLoaderService.queryClassLoaderById(testJobId, testJars).isPresent());
+ Assertions.assertEquals(
+ 0, classLoaderService.queryClassLoaderReferenceCount(testJobId, testJars));
+ testJar.delete();
+ }
+
+ /**
+ * Verifies that a failure before context publication releases the acquired classloader and is
+ * still reported to the master.
+ */
+ @Test
+ public void testDeployTaskHandlesFailureBeforeContextPublication() throws IOException {
+ TaskExecutionService taskExecutionService = Mockito.spy(server.getTaskExecutionService());
+ Mockito.doNothing()
+ .when(taskExecutionService)
+ .notifyTaskStatusToMaster(Mockito.any(), Mockito.any());
+ DefaultClassLoaderService classLoaderService =
+ (DefaultClassLoaderService) server.getClassLoaderService();
+
+ File testJar = File.createTempFile("failed-context-publication", ".jar");
+ testJar.deleteOnExit();
+ URL testJarUrl = testJar.toURI().toURL();
+ Set<URL> testJars = Collections.singleton(testJarUrl);
+ long testJobId = System.currentTimeMillis();
+ TaskGroupLocation location = new TaskGroupLocation(testJobId, 1, 1);
+ Task task = new ContextInitializationFailureTask();
+ TaskGroupImmutableInformation taskGroupImmutableInformation =
+ new TaskGroupImmutableInformation(
+ testJobId,
+ 1,
+ TaskGroupType.DEFAULT,
+ location,
+ "testDeployTaskHandlesFailureBeforeContextPublication",
+ Collections.singletonList(
+ nodeEngine.getSerializationService().toData(task)),
+ Collections.singletonList(testJars),
+ Collections.singletonList(emptySet()));
+
+ TaskDeployState taskDeployState =
+ taskExecutionService.deployTask(taskGroupImmutableInformation);
+
+ Assertions.assertEquals(TaskDeployState.success(), taskDeployState);
+ Assertions.assertThrows(
+ TaskGroupContextNotFoundException.class,
+ () -> taskExecutionService.getActiveExecutionContext(location));
+ Assertions.assertTrue(
+ classLoaderService.queryClassLoaderById(testJobId, testJars).isPresent());
+ Assertions.assertEquals(
+ 0, classLoaderService.queryClassLoaderReferenceCount(testJobId, testJars));
+ Mockito.verify(taskExecutionService, Mockito.timeout(5000))
+ .notifyTaskStatusToMaster(
+ Mockito.eq(location),
+ Mockito.argThat(state -> state.getExecutionState() == FAILED));
+ testJar.delete();
+ }
+
/** Test task execution time is the same as the timer timeout */
@Test
public void testCriticalCallTime() throws InterruptedException {
@@ -539,4 +640,22 @@ public class TaskExecutionServiceTest extends AbstractSeaTunnelServerTest {
Assertions.assertDoesNotThrow(() -> taskExecutionService.closeTimerFlushTask(unknown));
}
+
+ private static class ContextInitializationFailureTask implements Task {
+
+ @Override
+ public void setTaskExecutionContext(TaskExecutionContext taskExecutionContext) {
+ throw new IllegalStateException("context initialization failed");
+ }
+
+ @Override
+ public ProgressState call() {
+ return ProgressState.DONE;
+ }
+
+ @Override
+ public Long getTaskID() {
+ return 1L;
+ }
+ }
}