You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by an...@apache.org on 2023/05/04 12:47:50 UTC
[incubator-celeborn] branch main updated: [CELEBORN-551] Remove unnecessary ShuffleClient.get() (#1456)
This is an automated email from the ASF dual-hosted git repository.
angerszhuuuu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 783d4e5dc [CELEBORN-551] Remove unnecessary ShuffleClient.get() (#1456)
783d4e5dc is described below
commit 783d4e5dc5616c0656ed8c408b3e7bef36ad1f63
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Thu May 4 20:47:45 2023 +0800
[CELEBORN-551] Remove unnecessary ShuffleClient.get() (#1456)
---
.../spark/shuffle/celeborn/RssShuffleManager.java | 5 ++++-
.../spark/shuffle/celeborn/RssShuffleManager.java | 5 ++++-
.../org/apache/celeborn/client/ShuffleClient.java | 24 ----------------------
3 files changed, 8 insertions(+), 26 deletions(-)
diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index 3c610d881..b791e28b2 100644
--- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -102,7 +102,10 @@ public class RssShuffleManager implements ShuffleManager {
lifecycleManager = new LifecycleManager(appId, celebornConf);
rssShuffleClient =
ShuffleClient.get(
- lifecycleManager.self(), celebornConf, lifecycleManager.getUserIdentifier());
+ lifecycleManager.getRssMetaServiceHost(),
+ lifecycleManager.getRssMetaServicePort(),
+ celebornConf,
+ lifecycleManager.getUserIdentifier());
}
}
}
diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index 8eea44984..b279277ea 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -100,7 +100,10 @@ public class RssShuffleManager implements ShuffleManager {
lifecycleManager = new LifecycleManager(appId, celebornConf);
rssShuffleClient =
ShuffleClient.get(
- lifecycleManager.self(), celebornConf, lifecycleManager.getUserIdentifier());
+ lifecycleManager.getRssMetaServiceHost(),
+ lifecycleManager.getRssMetaServicePort(),
+ celebornConf,
+ lifecycleManager.getUserIdentifier());
}
}
}
diff --git a/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java b/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
index 2b5d07c5b..abd3399ba 100644
--- a/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
+++ b/client/src/main/java/org/apache/celeborn/client/ShuffleClient.java
@@ -51,30 +51,6 @@ public abstract class ShuffleClient {
protected ShuffleClient() {}
- public static ShuffleClient get(
- RpcEndpointRef driverRef, CelebornConf conf, UserIdentifier userIdentifier) {
- if (null == _instance || !initialized) {
- synchronized (ShuffleClient.class) {
- if (null == _instance) {
- // During the execution of Spark tasks, each task may be interrupted due to speculative
- // tasks. If the Task is interrupted while obtaining the ShuffleClient and the
- // ShuffleClient is building a singleton, it may cause the MetaServiceEndpoint to not be
- // assigned. An Executor will only construct a ShuffleClient singleton once. At this time,
- // when communicating with MetaService, it will cause a NullPointerException.
- _instance = new ShuffleClientImpl(conf, userIdentifier);
- _instance.setupMetaServiceRef(driverRef);
- initialized = true;
- } else if (!initialized) {
- _instance.shutdown();
- _instance = new ShuffleClientImpl(conf, userIdentifier);
- _instance.setupMetaServiceRef(driverRef);
- initialized = true;
- }
- }
- }
- return _instance;
- }
-
public static ShuffleClient get(
String driverHost, int port, CelebornConf conf, UserIdentifier userIdentifier) {
if (null == _instance || !initialized) {