You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/02/11 00:10:01 UTC
spark git commit: [SPARK-18717][SQL] Make code generation for Scala
Map work with immutable.Map also
Repository: spark
Updated Branches:
refs/heads/branch-2.1 7b5ea000e -> e580bb035
[SPARK-18717][SQL] Make code generation for Scala Map work with immutable.Map also
## What changes were proposed in this pull request?
Fixes compile errors in generated code when user has case class with a `scala.collections.immutable.Map` instead of a `scala.collections.Map`. Since ArrayBasedMapData.toScalaMap returns the immutable version we can make it work with both.
## How was this patch tested?
Additional unit tests.
Author: Andrew Ray <ra...@gmail.com>
Closes #16161 from aray/fix-map-codegen.
(cherry picked from commit 46d30ac4846b3ec94426cc482c42cff72ebd6d92)
Signed-off-by: Cheng Lian <li...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e580bb03
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e580bb03
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e580bb03
Branch: refs/heads/branch-2.1
Commit: e580bb035236dd92ade126af6bb98288d88179c4
Parents: 7b5ea00
Author: Andrew Ray <ra...@gmail.com>
Authored: Tue Dec 13 15:49:22 2016 +0800
Committer: Cheng Lian <li...@databricks.com>
Committed: Fri Feb 10 16:10:16 2017 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +-
.../test/scala/org/apache/spark/sql/DatasetSuite.scala | 12 ++++++++++++
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e580bb03/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 0aa21b9..fa1b900 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -342,7 +342,7 @@ object ScalaReflection extends ScalaReflection {
StaticInvoke(
ArrayBasedMapData.getClass,
- ObjectType(classOf[Map[_, _]]),
+ ObjectType(classOf[scala.collection.immutable.Map[_, _]]),
"toScalaMap",
keyData :: valueData :: Nil)
http://git-wip-us.apache.org/repos/asf/spark/blob/e580bb03/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index bdf6264..381652d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1063,8 +1063,20 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
// sizeInBytes is 2404280404, before the fix, it overflows to a negative number
assert(sizeInBytes > 0)
}
+
+ test("SPARK-18717: code generation works for both scala.collection.Map" +
+ " and scala.collection.imutable.Map") {
+ val ds = Seq(WithImmutableMap("hi", Map(42L -> "foo"))).toDS
+ checkDataset(ds.map(t => t), WithImmutableMap("hi", Map(42L -> "foo")))
+
+ val ds2 = Seq(WithMap("hi", Map(42L -> "foo"))).toDS
+ checkDataset(ds2.map(t => t), WithMap("hi", Map(42L -> "foo")))
+ }
}
+case class WithImmutableMap(id: String, map_test: scala.collection.immutable.Map[Long, String])
+case class WithMap(id: String, map_test: scala.collection.Map[Long, String])
+
case class Generic[T](id: T, value: Double)
case class OtherTuple(_1: String, _2: Int)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org