You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/12/19 22:35:20 UTC
[3/4] git commit: Add comment explaining collectPartitions's use
Add comment explaining collectPartitions's use
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/9cc3a6d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/9cc3a6d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/9cc3a6d3
Branch: refs/heads/master
Commit: 9cc3a6d3c0a64b80af77ae358c58d4b29b18c534
Parents: d3234f9
Author: Shivaram Venkataraman <sh...@eecs.berkeley.edu>
Authored: Thu Dec 19 11:49:17 2013 -0800
Committer: Shivaram Venkataraman <sh...@eecs.berkeley.edu>
Committed: Thu Dec 19 11:49:17 2013 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/9cc3a6d3/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
index 458d9dc..f344804 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
@@ -248,6 +248,8 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* Return an array that contains all of the elements in a specific partition of this RDD.
*/
def collectPartitions(partitionIds: Array[Int]): Array[JList[T]] = {
+ // This is useful for implementing `take` from other language frontends
+ // like Python where the data is serialized.
import scala.collection.JavaConversions._
val res = context.runJob(rdd, (it: Iterator[T]) => it.toArray, partitionIds, true)
res.map(x => new java.util.ArrayList(x.toSeq)).toArray