You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/08/08 20:10:14 UTC
git commit: [SPARK-2908] [SQL] JsonRDD.nullTypeToStringType does not
convert all NullType to StringType
Repository: spark
Updated Branches:
refs/heads/master 9016af3f2 -> 0489cee6b
[SPARK-2908] [SQL] JsonRDD.nullTypeToStringType does not convert all NullType to StringType
JIRA: https://issues.apache.org/jira/browse/SPARK-2908
Author: Yin Huai <hu...@cse.ohio-state.edu>
Closes #1840 from yhuai/SPARK-2908 and squashes the following commits:
86e833e [Yin Huai] Update test.
cb11759 [Yin Huai] nullTypeToStringType should check columns with the type of array of structs.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0489cee6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0489cee6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0489cee6
Branch: refs/heads/master
Commit: 0489cee6b24ca34f1adab03a75d157e04a9e06b7
Parents: 9016af3
Author: Yin Huai <hu...@cse.ohio-state.edu>
Authored: Fri Aug 8 11:10:11 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Fri Aug 8 11:10:11 2014 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/json/JsonRDD.scala | 4 +++-
.../test/scala/org/apache/spark/sql/json/JsonSuite.scala | 11 ++++++++---
.../scala/org/apache/spark/sql/json/TestJsonData.scala | 2 +-
3 files changed, 12 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0489cee6/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index a3d2a1c..1c0b03c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -109,7 +109,9 @@ private[sql] object JsonRDD extends Logging {
val newType = dataType match {
case NullType => StringType
case ArrayType(NullType, containsNull) => ArrayType(StringType, containsNull)
- case struct: StructType => nullTypeToStringType(struct)
+ case ArrayType(struct: StructType, containsNull) =>
+ ArrayType(nullTypeToStringType(struct), containsNull)
+ case struct: StructType =>nullTypeToStringType(struct)
case other: DataType => other
}
StructField(fieldName, newType, nullable)
http://git-wip-us.apache.org/repos/asf/spark/blob/0489cee6/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
index 75c0589..58b1e23 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
@@ -213,7 +213,8 @@ class JsonSuite extends QueryTest {
StructField("arrayOfStruct", ArrayType(
StructType(
StructField("field1", BooleanType, true) ::
- StructField("field2", StringType, true) :: Nil)), true) ::
+ StructField("field2", StringType, true) ::
+ StructField("field3", StringType, true) :: Nil)), true) ::
StructField("struct", StructType(
StructField("field1", BooleanType, true) ::
StructField("field2", DecimalType, true) :: Nil), true) ::
@@ -263,8 +264,12 @@ class JsonSuite extends QueryTest {
// Access elements of an array of structs.
checkAnswer(
- sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2] from jsonTable"),
- (true :: "str1" :: Nil, false :: null :: Nil, null) :: Nil
+ sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
+ "from jsonTable"),
+ (true :: "str1" :: null :: Nil,
+ false :: null :: null :: Nil,
+ null :: null :: null :: Nil,
+ null) :: Nil
)
// Access a struct and fields inside of it.
http://git-wip-us.apache.org/repos/asf/spark/blob/0489cee6/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala b/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala
index d0180f3..a88310b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/TestJsonData.scala
@@ -43,7 +43,7 @@ object TestJsonData {
"arrayOfDouble":[1.2, 1.7976931348623157E308, 4.9E-324, 2.2250738585072014E-308],
"arrayOfBoolean":[true, false, true],
"arrayOfNull":[null, null, null, null],
- "arrayOfStruct":[{"field1": true, "field2": "str1"}, {"field1": false}],
+ "arrayOfStruct":[{"field1": true, "field2": "str1"}, {"field1": false}, {"field3": null}],
"arrayOfArray1":[[1, 2, 3], ["str1", "str2"]],
"arrayOfArray2":[[1, 2, 3], [1.1, 2.1, 3.1]]
}""" :: Nil)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org