You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ad...@apache.org on 2014/10/29 01:59:21 UTC
git commit: [SPARK-4008] Fix "kryo with fold" in KryoSerializerSuite
Repository: spark
Updated Branches:
refs/heads/master 84e5da87e -> 1536d7033
[SPARK-4008] Fix "kryo with fold" in KryoSerializerSuite
`zeroValue` will be serialized by `spark.closure.serializer` but `spark.closure.serializer` only supports the default Java serializer. So it must not be `ClassWithoutNoArgConstructor`, which can not be serialized by the Java serializer.
This PR changed `zeroValue` to null and updated the test to make it work correctly.
Author: zsxwing <zs...@gmail.com>
Closes #2856 from zsxwing/SPARK-4008 and squashes the following commits:
51da655 [zsxwing] [SPARK-4008] Fix "kryo with fold" in KryoSerializerSuite
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1536d703
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1536d703
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1536d703
Branch: refs/heads/master
Commit: 1536d70331e9a4f5b5ea9dabfd72592ca1fc8e35
Parents: 84e5da8
Author: zsxwing <zs...@gmail.com>
Authored: Tue Oct 28 17:59:10 2014 -0700
Committer: Aaron Davidson <aa...@databricks.com>
Committed: Tue Oct 28 17:59:10 2014 -0700
----------------------------------------------------------------------
.../apache/spark/serializer/KryoSerializerSuite.scala | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1536d703/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 64ac6d2..a70f67a 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -201,12 +201,17 @@ class KryoSerializerSuite extends FunSuite with SharedSparkContext {
assert(control.sum === result)
}
- // TODO: this still doesn't work
- ignore("kryo with fold") {
+ test("kryo with fold") {
val control = 1 :: 2 :: Nil
+ // zeroValue must not be a ClassWithoutNoArgConstructor instance because it will be
+ // serialized by spark.closure.serializer but spark.closure.serializer only supports
+ // the default Java serializer.
val result = sc.parallelize(control, 2).map(new ClassWithoutNoArgConstructor(_))
- .fold(new ClassWithoutNoArgConstructor(10))((t1, t2) => new ClassWithoutNoArgConstructor(t1.x + t2.x)).x
- assert(10 + control.sum === result)
+ .fold(null)((t1, t2) => {
+ val t1x = if (t1 == null) 0 else t1.x
+ new ClassWithoutNoArgConstructor(t1x + t2.x)
+ }).x
+ assert(control.sum === result)
}
test("kryo with nonexistent custom registrator should fail") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org