You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/05/21 20:43:01 UTC
spark git commit: [SPARK-6416] [DOCS] RDD.fold() requires the
operator to be commutative
Repository: spark
Updated Branches:
refs/heads/master 4b7ff3092 -> 6e5340269
[SPARK-6416] [DOCS] RDD.fold() requires the operator to be commutative
Document current limitation of rdd.fold.
This does not resolve SPARK-6416 but just documents the issue.
CC JoshRosen
Author: Sean Owen <so...@cloudera.com>
Closes #6231 from srowen/SPARK-6416 and squashes the following commits:
9fef39f [Sean Owen] Add comment to other languages; reword to highlight the difference from non-distributed collections and to not suggest it is a bug that is to be fixed
da40d84 [Sean Owen] Document current limitation of rdd.fold.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e534026
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e534026
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e534026
Branch: refs/heads/master
Commit: 6e534026963e567f92743c5721de16325645223e
Parents: 4b7ff30
Author: Sean Owen <so...@cloudera.com>
Authored: Thu May 21 19:42:51 2015 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Thu May 21 19:42:51 2015 +0100
----------------------------------------------------------------------
.../scala/org/apache/spark/api/java/JavaRDDLike.scala | 13 ++++++++++---
core/src/main/scala/org/apache/spark/rdd/RDD.scala | 13 ++++++++++---
python/pyspark/rdd.py | 12 ++++++++++--
3 files changed, 30 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6e534026/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
index 8bf0627..74db764 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
@@ -386,9 +386,16 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
/**
* Aggregate the elements of each partition, and then the results for all the partitions, using a
- * given associative function and a neutral "zero value". The function op(t1, t2) is allowed to
- * modify t1 and return it as its result value to avoid object allocation; however, it should not
- * modify t2.
+ * given associative and commutative function and a neutral "zero value". The function
+ * op(t1, t2) is allowed to modify t1 and return it as its result value to avoid object
+ * allocation; however, it should not modify t2.
+ *
+ * This behaves somewhat differently from fold operations implemented for non-distributed
+ * collections in functional languages like Scala. This fold operation may be applied to
+ * partitions individually, and then fold those results into the final result, rather than
+ * apply the fold to each element sequentially in some defined ordering. For functions
+ * that are not commutative, the result may differ from that of a fold applied to a
+ * non-distributed collection.
*/
def fold(zeroValue: T)(f: JFunction2[T, T, T]): T =
rdd.fold(zeroValue)(f)
http://git-wip-us.apache.org/repos/asf/spark/blob/6e534026/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index f7fa37e..d772f03 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -1015,9 +1015,16 @@ abstract class RDD[T: ClassTag](
/**
* Aggregate the elements of each partition, and then the results for all the partitions, using a
- * given associative function and a neutral "zero value". The function op(t1, t2) is allowed to
- * modify t1 and return it as its result value to avoid object allocation; however, it should not
- * modify t2.
+ * given associative and commutative function and a neutral "zero value". The function
+ * op(t1, t2) is allowed to modify t1 and return it as its result value to avoid object
+ * allocation; however, it should not modify t2.
+ *
+ * This behaves somewhat differently from fold operations implemented for non-distributed
+ * collections in functional languages like Scala. This fold operation may be applied to
+ * partitions individually, and then fold those results into the final result, rather than
+ * apply the fold to each element sequentially in some defined ordering. For functions
+ * that are not commutative, the result may differ from that of a fold applied to a
+ * non-distributed collection.
*/
def fold(zeroValue: T)(op: (T, T) => T): T = withScope {
// Clone the zero value since we will also be serializing it as part of tasks
http://git-wip-us.apache.org/repos/asf/spark/blob/6e534026/python/pyspark/rdd.py
----------------------------------------------------------------------
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index 70db4bb..98a8ff8 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -813,13 +813,21 @@ class RDD(object):
def fold(self, zeroValue, op):
"""
Aggregate the elements of each partition, and then the results for all
- the partitions, using a given associative function and a neutral "zero
- value."
+ the partitions, using a given associative and commutative function and
+ a neutral "zero value."
The function C{op(t1, t2)} is allowed to modify C{t1} and return it
as its result value to avoid object allocation; however, it should not
modify C{t2}.
+ This behaves somewhat differently from fold operations implemented
+ for non-distributed collections in functional languages like Scala.
+ This fold operation may be applied to partitions individually, and then
+ fold those results into the final result, rather than apply the fold
+ to each element sequentially in some defined ordering. For functions
+ that are not commutative, the result may differ from that of a fold
+ applied to a non-distributed collection.
+
>>> from operator import add
>>> sc.parallelize([1, 2, 3, 4, 5]).fold(0, add)
15
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org