You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/02/21 01:46:23 UTC
git commit: Super minor: Add require for mergeCombiners in
combineByKey
Repository: incubator-spark
Updated Branches:
refs/heads/master 9e63f80e7 -> 3fede4831
Super minor: Add require for mergeCombiners in combineByKey
We changed the behavior in 0.9.0 from requiring that mergeCombiners be null when mapSideCombine was false to requiring that mergeCombiners *never* be null, for external sorting. This patch adds a require() to make this behavior change explicitly messaged rather than resulting in a NPE.
Author: Aaron Davidson <aa...@databricks.com>
Closes #623 from aarondav/master and squashes the following commits:
520b80c [Aaron Davidson] Super minor: Add require for mergeCombiners in combineByKey
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/3fede483
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/3fede483
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/3fede483
Branch: refs/heads/master
Commit: 3fede4831eeb7d36d4f8fa4aaa02ad0cc8b4b09e
Parents: 9e63f80
Author: Aaron Davidson <aa...@databricks.com>
Authored: Thu Feb 20 16:46:13 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Thu Feb 20 16:46:13 2014 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/3fede483/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index 15bec39..39c3a49 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -77,6 +77,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)])
partitioner: Partitioner,
mapSideCombine: Boolean = true,
serializerClass: String = null): RDD[(K, C)] = {
+ require(mergeCombiners != null, "mergeCombiners must be defined") // required as of Spark 0.9.0
if (getKeyClass().isArray) {
if (mapSideCombine) {
throw new SparkException("Cannot use map-side combining with array keys.")