You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by ch...@apache.org on 2024/01/05 12:58:52 UTC
(incubator-celeborn) branch main updated: Revert "[CELEBORN-1209][FOLLOW] Fix warning based on Spark Enabled ShuffleTracking after Spark 3.4"
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 21f5e6208 Revert "[CELEBORN-1209][FOLLOW] Fix warning based on Spark Enabled ShuffleTracking after Spark 3.4"
21f5e6208 is described below
commit 21f5e6208c12d416a72680357777872062e665f0
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Fri Jan 5 20:58:42 2024 +0800
Revert "[CELEBORN-1209][FOLLOW] Fix warning based on Spark Enabled ShuffleTracking after Spark 3.4"
This reverts commit 865f6f3d83347386c894b0fd45126fbddf2bb4b5.
---
.../apache/spark/shuffle/celeborn/SparkShuffleManager.java | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
index 35d847f46..264bf7766 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/SparkShuffleManager.java
@@ -94,16 +94,11 @@ public class SparkShuffleManager implements ShuffleManager {
+ "use Celeborn as Remote Shuffle Service to avoid performance degradation.",
SQLConf.LOCAL_SHUFFLE_READER_ENABLED().key());
}
- if ((Boolean) conf.get(package$.MODULE$.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED())) {
+ if (conf.getBoolean("spark.dynamicAllocation.shuffleTracking.enabled", false)) {
logger.warn(
- "Detected {} (default is {}) is enabled, it's highly recommended to disable it when "
- + "use Celeborn as Remote Shuffle Service to avoid performance degradation.",
- package$.MODULE$.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED().key(),
- package$
- .MODULE$
- .DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED()
- .defaultValue()
- .map(v -> (Boolean) v));
+ "Detected spark.dynamicAllocation.shuffleTracking.enabled (default is false) is enabled, "
+ + "it's highly recommended to disable it when use Celeborn as Remote Shuffle Service "
+ + "to avoid performance degradation.");
}
this.conf = conf;
this.isDriver = isDriver;