You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2016/10/06 20:48:20 UTC
spark git commit: [SPARK-15062][SQL] Backport fix list type infer
serializer issue
Repository: spark
Updated Branches:
refs/heads/branch-1.6 376545e4d -> d3890deb7
[SPARK-15062][SQL] Backport fix list type infer serializer issue
This backports https://github.com/apache/spark/commit/733cbaa3c0ff617a630a9d6937699db37ad2943b
to Branch 1.6. It's a pretty simple patch, and would be nice to have for Spark 1.6.3.
Unit tests
Author: Burak Yavuz <br...@gmail.com>
Closes #15380 from brkyvz/bp-SPARK-15062.
Signed-off-by: Michael Armbrust <mi...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d3890deb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d3890deb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d3890deb
Branch: refs/heads/branch-1.6
Commit: d3890deb7bb49dd5016c823a0de0429f73877d70
Parents: 376545e
Author: Burak Yavuz <br...@gmail.com>
Authored: Thu Oct 6 13:47:49 2016 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Thu Oct 6 13:48:02 2016 -0700
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/ScalaReflection.scala | 11 +++++++----
.../spark/sql/catalyst/ScalaReflectionSuite.scala | 14 ++++++++++++++
2 files changed, 21 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d3890deb/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 8722191..9cf400f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -485,6 +485,13 @@ object ScalaReflection extends ScalaReflection {
extractorFor(unwrapped, optType, newPath))
}
+ // Since List[_] also belongs to localTypeOf[Product], we put this case before
+ // "case t if t <:< localTypeOf[Product]" to make sure it will match to the
+ // case "localTypeOf[Seq[_]]"
+ case t if t <:< localTypeOf[Seq[_]] =>
+ val TypeRef(_, _, Seq(elementType)) = t
+ toCatalystArray(inputObject, elementType)
+
case t if t <:< localTypeOf[Product] =>
val params = getConstructorParameters(t)
val nonNullOutput = CreateNamedStruct(params.flatMap { case (fieldName, fieldType) =>
@@ -500,10 +507,6 @@ object ScalaReflection extends ScalaReflection {
val TypeRef(_, _, Seq(elementType)) = t
toCatalystArray(inputObject, elementType)
- case t if t <:< localTypeOf[Seq[_]] =>
- val TypeRef(_, _, Seq(elementType)) = t
- toCatalystArray(inputObject, elementType)
-
case t if t <:< localTypeOf[Map[_, _]] =>
val TypeRef(_, _, Seq(keyType, valueType)) = t
http://git-wip-us.apache.org/repos/asf/spark/blob/d3890deb/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index c2aace1..fc58699 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -21,7 +21,9 @@ import java.math.BigInteger
import java.sql.{Date, Timestamp}
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.expressions.{BoundReference, Literal, NewInstance}
import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
case class PrimitiveData(
intField: Int,
@@ -229,4 +231,16 @@ class ScalaReflectionSuite extends SparkFunSuite {
assert(anyTypes.forall(!_.isPrimitive))
assert(anyTypes === Seq(classOf[java.lang.Object], classOf[java.lang.Object]))
}
+
+ test("SPARK-15062: Get correct serializer for List[_]") {
+ val list = List(1, 2, 3)
+ val serializer = extractorsFor[List[Int]](BoundReference(
+ 0, ObjectType(list.getClass), nullable = false))
+ assert(serializer.children.size == 2)
+ assert(serializer.children.head.isInstanceOf[Literal])
+ assert(serializer.children.head.asInstanceOf[Literal].value === UTF8String.fromString("value"))
+ assert(serializer.children.last.isInstanceOf[NewInstance])
+ assert(serializer.children.last.asInstanceOf[NewInstance]
+ .cls.isInstanceOf[Class[org.apache.spark.sql.catalyst.util.GenericArrayData]])
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org