You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/01/31 00:40:31 UTC
spark git commit: [SPARK-5504] [sql] convertToCatalyst should support
nested arrays
Repository: spark
Updated Branches:
refs/heads/master 986977340 -> e643de42a
[SPARK-5504] [sql] convertToCatalyst should support nested arrays
After the recent refactoring, convertToCatalyst in ScalaReflection does not recurse on Arrays. It should.
The test suite modification made the test fail before the fix in ScalaReflection. The fix makes the test suite succeed.
CC: marmbrus
Author: Joseph K. Bradley <jo...@databricks.com>
Closes #4295 from jkbradley/SPARK-5504 and squashes the following commits:
6b7276d [Joseph K. Bradley] Fixed issue in ScalaReflection.convertToCatalyst with Arrays with non-primitive types. Modified test suite so it failed before the fix and works after the fix.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e643de42
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e643de42
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e643de42
Branch: refs/heads/master
Commit: e643de42a70834dc967664bd297b58fc91a998e7
Parents: 9869773
Author: Joseph K. Bradley <jo...@databricks.com>
Authored: Fri Jan 30 15:40:14 2015 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Fri Jan 30 15:40:14 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/ScalaReflection.scala | 6 +++++-
.../org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala | 8 ++++++--
2 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e643de42/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 4def65b..90646fd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -57,7 +57,11 @@ trait ScalaReflection {
case (obj, udt: UserDefinedType[_]) => udt.serialize(obj)
case (o: Option[_], _) => o.map(convertToCatalyst(_, dataType)).orNull
case (s: Seq[_], arrayType: ArrayType) => s.map(convertToCatalyst(_, arrayType.elementType))
- case (s: Array[_], arrayType: ArrayType) => s.toSeq
+ case (s: Array[_], arrayType: ArrayType) => if (arrayType.elementType.isPrimitive) {
+ s.toSeq
+ } else {
+ s.toSeq.map(convertToCatalyst(_, arrayType.elementType))
+ }
case (m: Map[_, _], mapType: MapType) => m.map { case (k, v) =>
convertToCatalyst(k, mapType.keyType) -> convertToCatalyst(v, mapType.valueType)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/e643de42/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index 4a66716..d0f547d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -64,7 +64,8 @@ case class ComplexData(
arrayFieldContainsNull: Seq[java.lang.Integer],
mapField: Map[Int, Long],
mapFieldValueContainsNull: Map[Int, java.lang.Long],
- structField: PrimitiveData)
+ structField: PrimitiveData,
+ nestedArrayField: Array[Array[Int]])
case class GenericData[A](
genericField: A)
@@ -158,7 +159,10 @@ class ScalaReflectionSuite extends FunSuite {
StructField("shortField", ShortType, nullable = false),
StructField("byteField", ByteType, nullable = false),
StructField("booleanField", BooleanType, nullable = false))),
- nullable = true))),
+ nullable = true),
+ StructField(
+ "nestedArrayField",
+ ArrayType(ArrayType(IntegerType, containsNull = false), containsNull = true)))),
nullable = true))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org