You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by gi...@apache.org on 2026/08/28 03:08:59 UTC

(seatunnel) 01/01: [Fix][Zeta] Release classloaders after failed task deployment (#11812)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-11812-a38f63109caf84ef3188003142b3ff98e83e3062
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit 1ff72ceb3d2a823a0fca64ec009984d75e846c28
Author: Goutam Adwant <86...@users.noreply.github.com>
AuthorDate: Fri Aug 28 03:08:35 2026 +0000

    [Fix][Zeta] Release classloaders after failed task deployment (#11812)
    
    Signed-off-by: Goutam Adwant <wo...@gmail.com>
    Signed-off-by: goutamadwant <wo...@gmail.com>
    Co-authored-by: DanielLeens <es...@outlook.com>
    Co-authored-by: David Zollo <da...@gmail.com>
---
 .../engine/server/TaskExecutionService.java        | 120 ++++++++++++++++-----
 .../engine/server/TaskExecutionServiceTest.java    | 119 ++++++++++++++++++++
 2 files changed, 210 insertions(+), 29 deletions(-)

diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
index eeef577ad2..7fd2be4c43 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/TaskExecutionService.java
@@ -104,6 +104,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.stream.Stream;
 
 import static com.hazelcast.jet.impl.util.ExceptionUtil.withTryCatch;
@@ -447,6 +448,8 @@ public class TaskExecutionService implements DynamicMetricsProvider {
                         "received deploying task executionId [%s]",
                         taskImmutableInfo.getExecutionId()));
         TaskGroup taskGroup = null;
+        // References owned by this deployment attempt until TaskGroupContext is published.
+        List<Collection<URL>> acquiredClassLoaderJars = new ArrayList<>();
         try {
             List<Set<ConnectorJarIdentifier>> connectorJarIdentifiersList =
                     taskImmutableInfo.getConnectorJarIdentifiers();
@@ -469,9 +472,11 @@ public class TaskExecutionService implements DynamicMetricsProvider {
                 } else if (!CollectionUtils.isEmpty(taskImmutableInfo.getJars().get(i))) {
                     jars = taskImmutableInfo.getJars().get(i);
                 }
+                List<URL> classLoaderJars = Lists.newArrayList(jars);
                 ClassLoader classLoader =
                         classLoaderService.getClassLoader(
-                                taskImmutableInfo.getJobId(), Lists.newArrayList(jars));
+                                taskImmutableInfo.getJobId(), classLoaderJars);
+                acquiredClassLoaderJars.add(classLoaderJars);
                 Task task;
                 if (jars.isEmpty()) {
                     task = nodeEngine.getSerializationService().toObject(taskData.get(i));
@@ -484,7 +489,7 @@ public class TaskExecutionService implements DynamicMetricsProvider {
                 }
                 tasks.add(task);
                 classLoaders.put(task.getTaskID(), classLoader);
-                taskJars.put(task.getTaskID(), jars);
+                taskJars.put(task.getTaskID(), classLoaderJars);
             }
             taskGroup =
                     TaskGroupUtils.createTaskGroup(
@@ -516,12 +521,30 @@ public class TaskExecutionService implements DynamicMetricsProvider {
                         classLoaderService.releaseClassLoader(
                                 taskImmutableInfo.getJobId(), entry.getValue());
                     }
+                    acquiredClassLoaderJars.clear();
                     return TaskDeployState.success();
                 }
-                deployLocalTask(taskGroup, classLoaders, taskJars);
+                AtomicBoolean classLoaderOwnershipTransferred = new AtomicBoolean();
+                deployLocalTask(
+                        taskGroup,
+                        classLoaders,
+                        taskJars,
+                        () -> classLoaderOwnershipTransferred.set(true),
+                        failure -> {
+                            releaseClassLoadersAfterFailedDeployment(
+                                    taskImmutableInfo.getJobId(), acquiredClassLoaderJars, failure);
+                            acquiredClassLoaderJars.clear();
+                        });
+                // Publication is a monotonic ownership transfer. The context may already have
+                // completed and left executionContexts by the time deployment returns.
+                if (classLoaderOwnershipTransferred.get()) {
+                    acquiredClassLoaderJars.clear();
+                }
                 return TaskDeployState.success();
             }
         } catch (Throwable t) {
+            releaseClassLoadersAfterFailedDeployment(
+                    taskImmutableInfo.getJobId(), acquiredClassLoaderJars, t);
             logger.severe(
                     String.format(
                             "TaskGroupID : %s  deploy error with Exception: %s",
@@ -533,6 +556,30 @@ public class TaskExecutionService implements DynamicMetricsProvider {
         }
     }
 
+    /**
+     * Releases classloader references acquired by the current deployment attempt.
+     *
+     * <p>This cleanup only releases references that have not been transferred to a published {@link
+     * TaskGroupContext}. Cleanup failures are added to the deployment failure so all references are
+     * attempted without replacing the original error.
+     */
+    private void releaseClassLoadersAfterFailedDeployment(
+            long jobId,
+            List<Collection<URL>> acquiredClassLoaderJars,
+            Throwable deploymentFailure) {
+        // Release in reverse acquisition order, matching the ownership stack built above.
+        for (int i = acquiredClassLoaderJars.size() - 1; i >= 0; i--) {
+            Collection<URL> jars = acquiredClassLoaderJars.get(i);
+            try {
+                classLoaderService.releaseClassLoader(jobId, jars);
+            } catch (Throwable cleanupFailure) {
+                deploymentFailure.addSuppressed(cleanupFailure);
+                logger.severe(
+                        "Release classloader after failed task deployment failed", cleanupFailure);
+            }
+        }
+    }
+
     /**
      * Deploys a task group locally. This method initializes the task group, creates execution
      * contexts, and submits tasks for execution based on the configured thread share mode.
@@ -546,7 +593,42 @@ public class TaskExecutionService implements DynamicMetricsProvider {
             @NonNull TaskGroup taskGroup,
             @NonNull ConcurrentHashMap<Long, ClassLoader> classLoaders,
             ConcurrentHashMap<Long, Collection<URL>> jars) {
+        return deployLocalTask(taskGroup, classLoaders, jars, () -> {}, failure -> {});
+    }
+
+    private PassiveCompletableFuture<TaskExecutionState> deployLocalTask(
+            @NonNull TaskGroup taskGroup,
+            @NonNull ConcurrentHashMap<Long, ClassLoader> classLoaders,
+            ConcurrentHashMap<Long, Collection<URL>> jars,
+            Runnable onContextPublished,
+            Consumer<Throwable> onFailureBeforeContextPublished) {
         CompletableFuture<TaskExecutionState> resultFuture = new CompletableFuture<>();
+        resultFuture.whenCompleteAsync(
+                withTryCatch(
+                        logger,
+                        (r, s) -> {
+                            if (s != null) {
+                                logger.severe(
+                                        String.format(
+                                                "Task %s complete with error %s",
+                                                taskGroup.getTaskGroupLocation(),
+                                                ExceptionUtils.getMessage(s)));
+                            }
+                            if (r == null) {
+                                r =
+                                        new TaskExecutionState(
+                                                taskGroup.getTaskGroupLocation(),
+                                                ExecutionState.FAILED,
+                                                s);
+                            }
+                            logger.info(
+                                    String.format(
+                                            "Task %s complete with state %s",
+                                            r.getTaskGroupLocation(), r.getExecutionState()));
+                            notifyTaskStatusToMaster(taskGroup.getTaskGroupLocation(), r);
+                        }),
+                MDCTracer.tracing(executorService));
+        boolean contextPublished = false;
         try {
             taskGroup.init();
             logger.info(
@@ -591,6 +673,8 @@ public class TaskExecutionService implements DynamicMetricsProvider {
             executionContexts.put(
                     taskGroup.getTaskGroupLocation(),
                     new TaskGroupContext(taskGroup, classLoaders, jars));
+            contextPublished = true;
+            onContextPublished.run();
             cancellationFutures.put(taskGroup.getTaskGroupLocation(), cancellationFuture);
             submitThreadShareTask(executionTracker, byCooperation.get(true));
             submitBlockingTask(executionTracker, byCooperation.get(false));
@@ -600,33 +684,11 @@ public class TaskExecutionService implements DynamicMetricsProvider {
                             "deploying TaskGroup %s success", taskGroup.getTaskGroupLocation()));
         } catch (Throwable t) {
             logger.severe(ExceptionUtils.getMessage(t));
+            if (!contextPublished) {
+                onFailureBeforeContextPublished.accept(t);
+            }
             resultFuture.completeExceptionally(t);
         }
-        resultFuture.whenCompleteAsync(
-                withTryCatch(
-                        logger,
-                        (r, s) -> {
-                            if (s != null) {
-                                logger.severe(
-                                        String.format(
-                                                "Task %s complete with error %s",
-                                                taskGroup.getTaskGroupLocation(),
-                                                ExceptionUtils.getMessage(s)));
-                            }
-                            if (r == null) {
-                                r =
-                                        new TaskExecutionState(
-                                                taskGroup.getTaskGroupLocation(),
-                                                ExecutionState.FAILED,
-                                                s);
-                            }
-                            logger.info(
-                                    String.format(
-                                            "Task %s complete with state %s",
-                                            r.getTaskGroupLocation(), r.getExecutionState()));
-                            notifyTaskStatusToMaster(taskGroup.getTaskGroupLocation(), r);
-                        }),
-                MDCTracer.tracing(executorService));
         return new PassiveCompletableFuture<>(resultFuture);
     }
 
@@ -637,7 +699,7 @@ public class TaskExecutionService implements DynamicMetricsProvider {
      * @param taskGroupLocation the location of the task group
      * @param taskExecutionState the execution state to report
      */
-    private void notifyTaskStatusToMaster(
+    void notifyTaskStatusToMaster(
             TaskGroupLocation taskGroupLocation, TaskExecutionState taskExecutionState) {
         long sleepTime = 1000;
         boolean notifyStateSuccess = false;
diff --git a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
index 3b6649d296..4c98fd977b 100644
--- a/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
+++ b/seatunnel-engine/seatunnel-engine-server/src/test/java/org/apache/seatunnel/engine/server/TaskExecutionServiceTest.java
@@ -21,12 +21,16 @@ import org.apache.seatunnel.shade.com.google.common.collect.Lists;
 
 import org.apache.seatunnel.engine.common.utils.PassiveCompletableFuture;
 import org.apache.seatunnel.engine.common.utils.concurrent.CompletableFuture;
+import org.apache.seatunnel.engine.core.classloader.DefaultClassLoaderService;
+import org.apache.seatunnel.engine.server.exception.TaskGroupContextNotFoundException;
 import org.apache.seatunnel.engine.server.execution.BlockTask;
 import org.apache.seatunnel.engine.server.execution.ExceptionTestTask;
 import org.apache.seatunnel.engine.server.execution.FixedCallTestTimeTask;
+import org.apache.seatunnel.engine.server.execution.ProgressState;
 import org.apache.seatunnel.engine.server.execution.StopTimeTestTask;
 import org.apache.seatunnel.engine.server.execution.Task;
 import org.apache.seatunnel.engine.server.execution.TaskDeployState;
+import org.apache.seatunnel.engine.server.execution.TaskExecutionContext;
 import org.apache.seatunnel.engine.server.execution.TaskExecutionState;
 import org.apache.seatunnel.engine.server.execution.TaskGroup;
 import org.apache.seatunnel.engine.server.execution.TaskGroupContext;
@@ -41,6 +45,7 @@ import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.RepeatedTest;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
 
 import com.hazelcast.flakeidgen.FlakeIdGenerator;
 import com.hazelcast.internal.serialization.Data;
@@ -54,6 +59,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ScheduledFuture;
@@ -234,6 +240,101 @@ public class TaskExecutionServiceTest extends AbstractSeaTunnelServerTest {
         console.delete();
     }
 
+    /**
+     * Verifies that a partially constructed task group does not retain classloader references when
+     * a later task cannot be deserialized.
+     */
+    @Test
+    public void testDeployTaskReleasesClassLoadersWhenDeserializationFails() throws IOException {
+        TaskExecutionService taskExecutionService = server.getTaskExecutionService();
+        DefaultClassLoaderService classLoaderService =
+                (DefaultClassLoaderService) server.getClassLoaderService();
+
+        File testJar = File.createTempFile("failed-deployment", ".jar");
+        testJar.deleteOnExit();
+        URL testJarUrl = testJar.toURI().toURL();
+        Set<URL> testJars = Collections.singleton(testJarUrl);
+        long testJobId = System.currentTimeMillis();
+        TestTask validTask = new TestTask(new AtomicBoolean(false), 300, true);
+        TaskGroupImmutableInformation taskGroupImmutableInformation =
+                new TaskGroupImmutableInformation(
+                        testJobId,
+                        1,
+                        TaskGroupType.INTERMEDIATE_BLOCKING_QUEUE,
+                        new TaskGroupLocation(testJobId, 1, 1),
+                        "testDeployTaskReleasesClassLoadersWhenDeserializationFails",
+                        Arrays.asList(
+                                nodeEngine.getSerializationService().toData(validTask),
+                                nodeEngine.getSerializationService().toData("not a task")),
+                        Arrays.asList(testJars, testJars),
+                        Arrays.asList(emptySet(), emptySet()));
+
+        TaskDeployState taskDeployState =
+                taskExecutionService.deployTask(taskGroupImmutableInformation);
+
+        Assertions.assertFalse(taskDeployState.isSuccess());
+        Assertions.assertThrows(
+                TaskGroupContextNotFoundException.class,
+                () ->
+                        taskExecutionService.getActiveExecutionContext(
+                                taskGroupImmutableInformation.getTaskGroupLocation()));
+        Assertions.assertTrue(
+                classLoaderService.queryClassLoaderById(testJobId, testJars).isPresent());
+        Assertions.assertEquals(
+                0, classLoaderService.queryClassLoaderReferenceCount(testJobId, testJars));
+        testJar.delete();
+    }
+
+    /**
+     * Verifies that a failure before context publication releases the acquired classloader and is
+     * still reported to the master.
+     */
+    @Test
+    public void testDeployTaskHandlesFailureBeforeContextPublication() throws IOException {
+        TaskExecutionService taskExecutionService = Mockito.spy(server.getTaskExecutionService());
+        Mockito.doNothing()
+                .when(taskExecutionService)
+                .notifyTaskStatusToMaster(Mockito.any(), Mockito.any());
+        DefaultClassLoaderService classLoaderService =
+                (DefaultClassLoaderService) server.getClassLoaderService();
+
+        File testJar = File.createTempFile("failed-context-publication", ".jar");
+        testJar.deleteOnExit();
+        URL testJarUrl = testJar.toURI().toURL();
+        Set<URL> testJars = Collections.singleton(testJarUrl);
+        long testJobId = System.currentTimeMillis();
+        TaskGroupLocation location = new TaskGroupLocation(testJobId, 1, 1);
+        Task task = new ContextInitializationFailureTask();
+        TaskGroupImmutableInformation taskGroupImmutableInformation =
+                new TaskGroupImmutableInformation(
+                        testJobId,
+                        1,
+                        TaskGroupType.DEFAULT,
+                        location,
+                        "testDeployTaskHandlesFailureBeforeContextPublication",
+                        Collections.singletonList(
+                                nodeEngine.getSerializationService().toData(task)),
+                        Collections.singletonList(testJars),
+                        Collections.singletonList(emptySet()));
+
+        TaskDeployState taskDeployState =
+                taskExecutionService.deployTask(taskGroupImmutableInformation);
+
+        Assertions.assertEquals(TaskDeployState.success(), taskDeployState);
+        Assertions.assertThrows(
+                TaskGroupContextNotFoundException.class,
+                () -> taskExecutionService.getActiveExecutionContext(location));
+        Assertions.assertTrue(
+                classLoaderService.queryClassLoaderById(testJobId, testJars).isPresent());
+        Assertions.assertEquals(
+                0, classLoaderService.queryClassLoaderReferenceCount(testJobId, testJars));
+        Mockito.verify(taskExecutionService, Mockito.timeout(5000))
+                .notifyTaskStatusToMaster(
+                        Mockito.eq(location),
+                        Mockito.argThat(state -> state.getExecutionState() == FAILED));
+        testJar.delete();
+    }
+
     /** Test task execution time is the same as the timer timeout */
     @Test
     public void testCriticalCallTime() throws InterruptedException {
@@ -539,4 +640,22 @@ public class TaskExecutionServiceTest extends AbstractSeaTunnelServerTest {
 
         Assertions.assertDoesNotThrow(() -> taskExecutionService.closeTimerFlushTask(unknown));
     }
+
+    private static class ContextInitializationFailureTask implements Task {
+
+        @Override
+        public void setTaskExecutionContext(TaskExecutionContext taskExecutionContext) {
+            throw new IllegalStateException("context initialization failed");
+        }
+
+        @Override
+        public ProgressState call() {
+            return ProgressState.DONE;
+        }
+
+        @Override
+        public Long getTaskID() {
+            return 1L;
+        }
+    }
 }