You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/01/03 06:28:12 UTC
[incubator-celeborn] branch main updated: [CELEBORN-185][SPARK] Can't release shuffle data if rss fallback to nss (#1133)
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 41b1fa46 [CELEBORN-185][SPARK] Can't release shuffle data if rss fallback to nss (#1133)
41b1fa46 is described below
commit 41b1fa46d33b993dcd911fea77e4dc435a017b62
Author: jxysoft <xi...@gmail.com>
AuthorDate: Tue Jan 3 14:28:09 2023 +0800
[CELEBORN-185][SPARK] Can't release shuffle data if rss fallback to nss (#1133)
Co-authored-by: xianyao.jiang <xi...@antfin.com>
---
.../main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java | 1 +
.../main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java | 1 +
2 files changed, 2 insertions(+)
diff --git a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index 34feca61..00ed42e2 100644
--- a/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-2/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -172,6 +172,7 @@ public class RssShuffleManager implements ShuffleManager {
"Unrecognized shuffle write mode!" + celebornConf.shuffleWriterMode());
}
} else {
+ sortShuffleIds.add(handle.shuffleId());
return sortShuffleManager().getWriter(handle, mapId, context);
}
} catch (IOException e) {
diff --git a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
index 78a31562..d2b1769e 100644
--- a/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
+++ b/client-spark/spark-3/src/main/java/org/apache/spark/shuffle/celeborn/RssShuffleManager.java
@@ -170,6 +170,7 @@ public class RssShuffleManager implements ShuffleManager {
"Unrecognized shuffle write mode!" + celebornConf.shuffleWriterMode());
}
} else {
+ sortShuffleIds.add(handle.shuffleId());
return sortShuffleManager().getWriter(handle, mapId, context, metrics);
}
} catch (IOException e) {