You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/01/12 10:04:53 UTC
spark git commit: [SPARK-23025][SQL] Support Null type in scala
reflection
Repository: spark
Updated Branches:
refs/heads/master a7d98d53c -> 505086806
[SPARK-23025][SQL] Support Null type in scala reflection
## What changes were proposed in this pull request?
Add support for `Null` type in the `schemaFor` method for Scala reflection.
## How was this patch tested?
Added UT
Author: Marco Gaido <ma...@gmail.com>
Closes #20219 from mgaido91/SPARK-23025.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/50508680
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/50508680
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/50508680
Branch: refs/heads/master
Commit: 505086806997b4331d4a8c2fc5e08345d869a23c
Parents: a7d98d5
Author: Marco Gaido <ma...@gmail.com>
Authored: Fri Jan 12 18:04:44 2018 +0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Fri Jan 12 18:04:44 2018 +0800
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/ScalaReflection.scala | 4 ++++
.../apache/spark/sql/catalyst/ScalaReflectionSuite.scala | 9 +++++++++
.../src/test/scala/org/apache/spark/sql/DatasetSuite.scala | 5 +++++
3 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/50508680/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 65040f1..9a4bf00 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -63,6 +63,7 @@ object ScalaReflection extends ScalaReflection {
private def dataTypeFor(tpe: `Type`): DataType = cleanUpReflectionObjects {
tpe.dealias match {
+ case t if t <:< definitions.NullTpe => NullType
case t if t <:< definitions.IntTpe => IntegerType
case t if t <:< definitions.LongTpe => LongType
case t if t <:< definitions.DoubleTpe => DoubleType
@@ -712,6 +713,9 @@ object ScalaReflection extends ScalaReflection {
/** Returns a catalyst DataType and its nullability for the given Scala Type using reflection. */
def schemaFor(tpe: `Type`): Schema = cleanUpReflectionObjects {
tpe.dealias match {
+ // this must be the first case, since all objects in scala are instances of Null, therefore
+ // Null type would wrongly match the first of them, which is Option as of now
+ case t if t <:< definitions.NullTpe => Schema(NullType, nullable = true)
case t if t.typeSymbol.annotations.exists(_.tree.tpe =:= typeOf[SQLUserDefinedType]) =>
val udt = getClassFromType(t).getAnnotation(classOf[SQLUserDefinedType]).udt().newInstance()
Schema(udt, nullable = true)
http://git-wip-us.apache.org/repos/asf/spark/blob/50508680/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index 23e866c..8c3db48 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -356,4 +356,13 @@ class ScalaReflectionSuite extends SparkFunSuite {
assert(deserializerFor[Int].isInstanceOf[AssertNotNull])
assert(!deserializerFor[String].isInstanceOf[AssertNotNull])
}
+
+ test("SPARK-23025: schemaFor should support Null type") {
+ val schema = schemaFor[(Int, Null)]
+ assert(schema === Schema(
+ StructType(Seq(
+ StructField("_1", IntegerType, nullable = false),
+ StructField("_2", NullType, nullable = true))),
+ nullable = true))
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/50508680/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index d535896..54893c1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1441,6 +1441,11 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
assert(e.getCause.isInstanceOf[NullPointerException])
}
}
+
+ test("SPARK-23025: Add support for null type in scala reflection") {
+ val data = Seq(("a", null))
+ checkDataset(data.toDS(), data: _*)
+ }
}
case class SingleData(id: Int)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org