You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by an...@apache.org on 2023/05/04 12:47:50 UTC

[incubator-celeborn] branch main updated: [CELEBORN-551] Remove unnecessary ShuffleClient.get() (#1456)

This is an automated email from the ASF dual-hosted git repository.

angerszhuuuu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 783d4e5dc [CELEBORN-551] Remove unnecessary ShuffleClient.get() (#1456)
783d4e5dc is described below

commit 783d4e5dc5616c0656ed8c408b3e7bef36ad1f63
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Thu May 4 20:47:45 2023 +0800

    [CELEBORN-551] Remove unnecessary ShuffleClient.get() (#1456)
---
 .../spark/shuffle/celeborn/RssShuffleManager.java  |  5 ++++-
 .../spark/shuffle/celeborn/RssShuffleManager.java  |  5 ++++-
 .../org/apache/celeborn/client/ShuffleClient.java  | 24 ----------------------
 3 files changed, 8 insertions(+), 26 deletions(-)

diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index 3c610d881..b791e28b2 100644
--- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -102,7 +102,10 @@ public class RssShuffleManager implements ShuffleManager {
           lifecycleManager = new LifecycleManager(appId, celebornConf);
           rssShuffleClient =
               ShuffleClient.get(
-                  lifecycleManager.self(), celebornConf, lifecycleManager.getUserIdentifier());
+                  lifecycleManager.getRssMetaServiceHost(),
+                  lifecycleManager.getRssMetaServicePort(),
+                  celebornConf,
+                  lifecycleManager.getUserIdentifier());
         }
       }
     }
diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index 8eea44984..b279277ea 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -100,7 +100,10 @@ public class RssShuffleManager implements ShuffleManager {
           lifecycleManager = new LifecycleManager(appId, celebornConf);
           rssShuffleClient =
               ShuffleClient.get(
-                  lifecycleManager.self(), celebornConf, lifecycleManager.getUserIdentifier());
+                  lifecycleManager.getRssMetaServiceHost(),
+                  lifecycleManager.getRssMetaServicePort(),
+                  celebornConf,
+                  lifecycleManager.getUserIdentifier());
         }
       }
     }
diff --git a/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java b/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
index 2b5d07c5b..abd3399ba 100644
--- a/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
+++ b/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
@@ -51,30 +51,6 @@ public abstract class ShuffleClient {
 
   protected ShuffleClient() {}
 
-  public static ShuffleClient get(
-      RpcEndpointRef driverRef, CelebornConf conf, UserIdentifier userIdentifier) {
-    if (null == _instance || !initialized) {
-      synchronized (ShuffleClient.class) {
-        if (null == _instance) {
-          // During the execution of Spark tasks, each task may be interrupted due to speculative
-          // tasks. If the Task is interrupted while obtaining the ShuffleClient and the
-          // ShuffleClient is building a singleton, it may cause the MetaServiceEndpoint to not be
-          // assigned. An Executor will only construct a ShuffleClient singleton once. At this time,
-          // when communicating with MetaService, it will cause a NullPointerException.
-          _instance = new ShuffleClientImpl(conf, userIdentifier);
-          _instance.setupMetaServiceRef(driverRef);
-          initialized = true;
-        } else if (!initialized) {
-          _instance.shutdown();
-          _instance = new ShuffleClientImpl(conf, userIdentifier);
-          _instance.setupMetaServiceRef(driverRef);
-          initialized = true;
-        }
-      }
-    }
-    return _instance;
-  }
-
   public static ShuffleClient get(
       String driverHost, int port, CelebornConf conf, UserIdentifier userIdentifier) {
     if (null == _instance || !initialized) {