You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by an...@apache.org on 2023/04/27 06:16:35 UTC
[incubator-celeborn] branch main updated: [CELEBORN-560][BUG] Rerun task in spark later then RSS stageEnd cause NPE then job failed (#1466)
This is an automated email from the ASF dual-hosted git repository.
angerszhuuuu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new ce21a738a [CELEBORN-560][BUG] Rerun task in spark later then RSS stageEnd cause NPE then job failed (#1466)
ce21a738a is described below
commit ce21a738a92e0fdadddf7194cea4d9020d7c1250
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Thu Apr 27 14:16:32 2023 +0800
[CELEBORN-560][BUG] Rerun task in spark later then RSS stageEnd cause NPE then job failed (#1466)
---
.../src/main/scala/org/apache/celeborn/client/LifecycleManager.scala | 4 ----
1 file changed, 4 deletions(-)
diff --git a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
index a82ea48f8..54d7ed07b 100644
--- a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala
@@ -569,9 +569,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
}
if (commitManager.tryFinalCommit(shuffleId)) {
- // release resources and clear worker info
- shuffleAllocatedWorkers.remove(shuffleId)
-
requestReleaseSlots(
rssHARetryClient,
ReleaseSlots(applicationId, shuffleId, List.empty.asJava, List.empty.asJava))
@@ -631,7 +628,6 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin
if (shuffleResourceExists(shuffleId)) {
logWarning(s"Partition exists for shuffle $shuffleId, " +
"maybe caused by task rerun or speculative.")
- shuffleAllocatedWorkers.remove(shuffleId)
requestReleaseSlots(
rssHARetryClient,
ReleaseSlots(appId, shuffleId, List.empty.asJava, List.empty.asJava))