You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by an...@apache.org on 2023/02/08 09:16:34 UTC
[incubator-celeborn] branch main updated: [CELEBORN-239][FOLLOWUP] PUSH_DATA_TIMEOUT_MASTER/SLAVE should support convert through RPC (#1211)
This is an automated email from the ASF dual-hosted git repository.
angerszhuuuu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new dae58a664 [CELEBORN-239][FOLLOWUP] PUSH_DATA_TIMEOUT_MASTER/SLAVE should support convert through RPC (#1211)
dae58a664 is described below
commit dae58a664c695544d1ac8b4992933d8b7c95be18
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Wed Feb 8 17:16:29 2023 +0800
[CELEBORN-239][FOLLOWUP] PUSH_DATA_TIMEOUT_MASTER/SLAVE should support convert through RPC (#1211)
* [CELEBORN-239][FOLLOWUP] PUSH_DATA_TIMEOUT_MASTER/SLAVE should support convert through RP
---
.../org/apache/celeborn/common/util/Utils.scala | 4 ++++
.../celeborn/tests/spark/PushDataTimeoutTest.scala | 20 ++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
index 31a716ddb..0926ac88f 100644
--- a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
@@ -902,6 +902,10 @@ object Utils extends Logging {
StatusCode.PUSH_DATA_CONNECTION_EXCEPTION_MASTER
case 41 =>
StatusCode.PUSH_DATA_CONNECTION_EXCEPTION_SLAVE
+ case 42 =>
+ StatusCode.PUSH_DATA_TIMEOUT_MASTER
+ case 43 =>
+ StatusCode.PUSH_DATA_TIMEOUT_SLAVE
case _ =>
null
}
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/PushDataTimeoutTest.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/PushDataTimeoutTest.scala
index d7cab66fd..66700ff45 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/PushDataTimeoutTest.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/PushDataTimeoutTest.scala
@@ -50,6 +50,7 @@ class PushDataTimeoutTest extends AnyFunSuite
val sparkConf = new SparkConf().setAppName("rss-demo").setMaster("local[4]")
.set("spark.celeborn.push.data.timeout", "5s")
.set("spark.celeborn.push.replicate.enabled", enabled)
+ .set("spark.celeborn.client.blacklistSlave.enabled", "false")
val sparkSession = SparkSession.builder().config(sparkConf).getOrCreate()
val combineResult = combine(sparkSession)
val groupbyResult = groupBy(sparkSession)
@@ -76,4 +77,23 @@ class PushDataTimeoutTest extends AnyFunSuite
ShuffleClient.reset()
}
}
+
+ test("celeborn spark integration test - pushdata timeout will add to balcklist") {
+ val sparkConf = new SparkConf().setAppName("rss-demo").setMaster("local[4]")
+ .set("spark.celeborn.push.data.timeout", "5s")
+ .set("spark.celeborn.push.replicate.enabled", "true")
+ .set("spark.celeborn.client.blacklistSlave.enabled", "true")
+ val rssSparkSession = SparkSession.builder()
+ .config(updateSparkConf(sparkConf, false)).getOrCreate()
+ try {
+ combine(rssSparkSession)
+ } catch {
+ case e: Exception =>
+ e.printStackTrace()
+ e.getMessage.concat("Revive Failed in retry push merged data for location")
+ }
+
+ rssSparkSession.stop()
+ ShuffleClient.reset()
+ }
}