[incubator-celeborn] branch branch-0.2 updated: [CELEBORN-472] Support using Celeborn in the scenario of switching multiple SparkContexts in the same process (#1379)

[ch...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch branch-0.2
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/branch-0.2 by this push:
     new b6fbe72d8 [CELEBORN-472] Support using Celeborn in the scenario of switching multiple SparkContexts in the same process (#1379)
b6fbe72d8 is described below

commit b6fbe72d84de4bd71f8a9b928a6cf151eaadf484
Author: Kerwin Zhang <xi...@alibaba-inc.com>
AuthorDate: Mon Mar 27 16:10:34 2023 +0800

    [CELEBORN-472] Support using Celeborn in the scenario of switching multiple SparkContexts in the same process (#1379)
---
 .../java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index d3d2b0bc5..304ebea6d 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -140,12 +140,16 @@ public class RssShuffleManager implements ShuffleManager {
   public void stop() {
     if (rssShuffleClient != null) {
       rssShuffleClient.shutdown();
+      ShuffleClient.reset();
+      rssShuffleClient = null;
     }
     if (lifecycleManager != null) {
       lifecycleManager.stop();
+      lifecycleManager = null;
     }
     if (sortShuffleManager() != null) {
       sortShuffleManager().stop();
+      _sortShuffleManager = null;
     }
   }