You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/11/05 06:29:28 UTC
spark git commit: [SPARK-18189][SQL][FOLLOWUP] Move test from
ReplSuite to prevent java.lang.ClassCircularityError
Repository: spark
Updated Branches:
refs/heads/branch-2.0 399597b04 -> 8b99e204a
[SPARK-18189][SQL][FOLLOWUP] Move test from ReplSuite to prevent java.lang.ClassCircularityError
## What changes were proposed in this pull request?
Move the test which is causing java.lang.ClassCircularityError from ReplSuite to DatasetSuite.
## How was this patch tested?
> build/mvn -DskipTests -Phadoop-2.3 -Pyarn -Phive -Phive-thriftserver -Pkinesis-asl -Pmesos clean package
> build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.repl.ReplSuite test
Author: Ergin Seyfe <es...@fb.com>
Closes #15774 from seyfe/fix_replsuite_test_error_branch2.0.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8b99e204
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8b99e204
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8b99e204
Branch: refs/heads/branch-2.0
Commit: 8b99e204a9a056fd071f9bd75f3e0a29f90bccc0
Parents: 399597b
Author: Ergin Seyfe <es...@fb.com>
Authored: Fri Nov 4 23:29:20 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Fri Nov 4 23:29:20 2016 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/repl/ReplSuite.scala | 17 -----------------
.../scala/org/apache/spark/sql/DatasetSuite.scala | 13 +++++++++++++
2 files changed, 13 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8b99e204/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index 8deafe3..f7d7a4f 100644
--- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -473,21 +473,4 @@ class ReplSuite extends SparkFunSuite {
assertDoesNotContain("AssertionError", output)
assertDoesNotContain("Exception", output)
}
-
- test("SPARK-18189: Fix serialization issue in KeyValueGroupedDataset") {
- val resultValue = 12345
- val output = runInterpreter("local",
- s"""
- |val keyValueGrouped = Seq((1, 2), (3, 4)).toDS().groupByKey(_._1)
- |val mapGroups = keyValueGrouped.mapGroups((k, v) => (k, 1))
- |val broadcasted = sc.broadcast($resultValue)
- |
- |// Using broadcast triggers serialization issue in KeyValueGroupedDataset
- |val dataset = mapGroups.map(_ => broadcasted.value)
- |dataset.collect()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains(s": Array[Int] = Array($resultValue, $resultValue)", output)
- }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/8b99e204/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index f897cfb..6113e5d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -882,6 +882,19 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
df.withColumn("b", expr("0")).as[ClassData]
.groupByKey(_.a).flatMapGroups { case (x, iter) => List[Int]() })
}
+
+ // This is moved from ReplSuite to prevent java.lang.ClassCircularityError.
+ test("SPARK-18189: Fix serialization issue in KeyValueGroupedDataset") {
+ val resultValue = 12345
+ val keyValueGrouped = Seq((1, 2), (3, 4)).toDS().groupByKey(_._1)
+ val mapGroups = keyValueGrouped.mapGroups((k, v) => (k, 1))
+ val broadcasted = spark.sparkContext.broadcast(resultValue)
+
+ // Using broadcast triggers serialization issue in KeyValueGroupedDataset
+ val dataset = mapGroups.map(_ => broadcasted.value)
+
+ assert(dataset.collect() sameElements Array(resultValue, resultValue))
+ }
}
case class Generic[T](id: T, value: Double)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org