You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/10/06 02:57:07 UTC
git commit: [SPARK-3776][SQL] Wrong conversion to Catalyst for
Option[Product]
Repository: spark
Updated Branches:
refs/heads/master 34b97a067 -> 90897ea5f
[SPARK-3776][SQL] Wrong conversion to Catalyst for Option[Product]
Author: Renat Yusupov <re...@2gis.ru>
Closes #2641 from r3natko/feature/catalyst_option and squashes the following commits:
55d0c06 [Renat Yusupov] [SQL] SPARK-3776: Wrong conversion to Catalyst for Option[Product]
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/90897ea5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/90897ea5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/90897ea5
Branch: refs/heads/master
Commit: 90897ea5f24b03c9f3455a62c7f68b3d3f0435ad
Parents: 34b97a0
Author: Renat Yusupov <re...@2gis.ru>
Authored: Sun Oct 5 17:56:24 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Sun Oct 5 17:56:34 2014 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/ScalaReflection.scala | 2 +-
.../sql/catalyst/ScalaReflectionSuite.scala | 21 +++++++++++++++++---
2 files changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/90897ea5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 88a8fa7..b3ae8e6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -33,7 +33,7 @@ object ScalaReflection {
/** Converts Scala objects to catalyst rows / types */
def convertToCatalyst(a: Any): Any = a match {
- case o: Option[_] => o.orNull
+ case o: Option[_] => o.map(convertToCatalyst).orNull
case s: Seq[_] => s.map(convertToCatalyst)
case m: Map[_, _] => m.map { case (k, v) => convertToCatalyst(k) -> convertToCatalyst(v) }
case p: Product => new GenericRow(p.productIterator.map(convertToCatalyst).toArray)
http://git-wip-us.apache.org/repos/asf/spark/blob/90897ea5/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index 428607d..488e373 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -53,7 +53,8 @@ case class OptionalData(
floatField: Option[Float],
shortField: Option[Short],
byteField: Option[Byte],
- booleanField: Option[Boolean])
+ booleanField: Option[Boolean],
+ structField: Option[PrimitiveData])
case class ComplexData(
arrayField: Seq[Int],
@@ -100,7 +101,7 @@ class ScalaReflectionSuite extends FunSuite {
nullable = true))
}
- test("optinal data") {
+ test("optional data") {
val schema = schemaFor[OptionalData]
assert(schema === Schema(
StructType(Seq(
@@ -110,7 +111,8 @@ class ScalaReflectionSuite extends FunSuite {
StructField("floatField", FloatType, nullable = true),
StructField("shortField", ShortType, nullable = true),
StructField("byteField", ByteType, nullable = true),
- StructField("booleanField", BooleanType, nullable = true))),
+ StructField("booleanField", BooleanType, nullable = true),
+ StructField("structField", schemaFor[PrimitiveData].dataType, nullable = true))),
nullable = true))
}
@@ -228,4 +230,17 @@ class ScalaReflectionSuite extends FunSuite {
assert(ArrayType(IntegerType) === typeOfObject3(Seq(1, 2, 3)))
assert(ArrayType(ArrayType(IntegerType)) === typeOfObject3(Seq(Seq(1,2,3))))
}
+
+ test("convert PrimitiveData to catalyst") {
+ val data = PrimitiveData(1, 1, 1, 1, 1, 1, true)
+ val convertedData = Seq(1, 1.toLong, 1.toDouble, 1.toFloat, 1.toShort, 1.toByte, true)
+ assert(convertToCatalyst(data) === convertedData)
+ }
+
+ test("convert Option[Product] to catalyst") {
+ val primitiveData = PrimitiveData(1, 1, 1, 1, 1, 1, true)
+ val data = OptionalData(Some(1), Some(1), Some(1), Some(1), Some(1), Some(1), Some(true), Some(primitiveData))
+ val convertedData = Seq(1, 1.toLong, 1.toDouble, 1.toFloat, 1.toShort, 1.toByte, true, convertToCatalyst(primitiveData))
+ assert(convertToCatalyst(data) === convertedData)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org