You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2020/03/19 03:14:29 UTC
[spark] branch branch-3.0 updated: [SPARK-30667][CORE] Change
BarrierTaskContext allGather method return type
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new f45be8b [SPARK-30667][CORE] Change BarrierTaskContext allGather method return type
f45be8b is described below
commit f45be8b19ba045889bfa54a0e4b8dce54c4dd910
Author: sarthfrey-db <sa...@databricks.com>
AuthorDate: Thu Mar 19 12:12:39 2020 +0900
[SPARK-30667][CORE] Change BarrierTaskContext allGather method return type
This PR proposes that we change the return type of the `BarrierTaskContext.allGather` method to `Array[String]` instead of `ArrayBuffer[String]` since it is immutable. Based on discussion in #27640. cc zhengruifeng srowen
Closes #27951 from sarthfrey/all-gather-api.
Authored-by: sarthfrey-db <sa...@databricks.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
(cherry picked from commit 6fd3138e9c3c1d703f2a66bcdf17555803547774)
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
core/src/main/scala/org/apache/spark/BarrierTaskContext.scala | 4 ++--
core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala | 2 +-
.../scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 2263538..0c2ceb1 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -217,11 +217,11 @@ class BarrierTaskContext private[spark] (
*/
@Experimental
@Since("3.0.0")
- def allGather(message: String): ArrayBuffer[String] = {
+ def allGather(message: String): Array[String] = {
val json = runBarrier(RequestMethod.ALL_GATHER, message)
val jsonArray = parse(json)
implicit val formats = DefaultFormats
- ArrayBuffer(jsonArray.extract[Array[String]]: _*)
+ jsonArray.extract[Array[String]]
}
/**
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index fa8bf0f..06c9446 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -420,7 +420,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
context.asInstanceOf[BarrierTaskContext].barrier()
result = BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS
case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
- val messages: ArrayBuffer[String] = context.asInstanceOf[BarrierTaskContext].allGather(
+ val messages: Array[String] = context.asInstanceOf[BarrierTaskContext].allGather(
message
)
result = compact(render(JArray(
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 33594c0..0dd8be7 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -65,7 +65,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
Thread.sleep(Random.nextInt(1000))
// Pass partitionId message in
val message: String = context.partitionId().toString
- val messages: ArrayBuffer[String] = context.allGather(message)
+ val messages: Array[String] = context.allGather(message)
messages.toList.iterator
}
// Take a sorted list of all the partitionId messages
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org