(incubator-celeborn) branch main updated: [CELEBORN-1096] Avoid initializing SortShuffleManager when stop

[fc...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

fchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 70366edb7 [CELEBORN-1096] Avoid initializing SortShuffleManager when stop
70366edb7 is described below

commit 70366edb7cef59ff27bdde888b747a897b1f0a2c
Author: sychen <sy...@ctrip.com>
AuthorDate: Mon Oct 30 14:21:06 2023 +0800

    [CELEBORN-1096] Avoid initializing SortShuffleManager when stop
    
    ### What changes were proposed in this pull request?
    
    ### Why are the changes needed?
    If there is no fallback to the Spark SortShuffleManager, it will be initialized once when stopping, and then stopped again. This is not necessary.
    
    ### Does this PR introduce _any_ user-facing change?
    
    ### How was this patch tested?
    
    Closes #2052 from cxzl25/CELEBORN-1096.
    
    Authored-by: sychen <sy...@ctrip.com>
    Signed-off-by: Fu Chen <cf...@gmail.com>
---
 .../java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java  | 5 +++--
 .../java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java  | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
index 74e983c2a..071810ad2 100644
--- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
+++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
@@ -167,8 +167,9 @@ public class SparkShuffleManager implements ShuffleManager {
     if (lifecycleManager != null) {
       lifecycleManager.stop();
     }
-    if (sortShuffleManager() != null) {
-      sortShuffleManager().stop();
+    if (_sortShuffleManager != null) {
+      _sortShuffleManager.stop();
+      _sortShuffleManager = null;
     }
   }
 
diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
index c0d3bd286..aa1c1a2c2 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
@@ -197,8 +197,8 @@ public class SparkShuffleManager implements ShuffleManager {
       lifecycleManager.stop();
       lifecycleManager = null;
     }
-    if (sortShuffleManager() != null) {
-      sortShuffleManager().stop();
+    if (_sortShuffleManager != null) {
+      _sortShuffleManager.stop();
       _sortShuffleManager = null;
     }
   }