You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by da...@apache.org on 2016/04/13 02:26:42 UTC
spark git commit: [SPARK-14578] [SQL] Fix codegen for
CreateExternalRow with nested wide schema
Repository: spark
Updated Branches:
refs/heads/master d187e7dea -> 372baf047
[SPARK-14578] [SQL] Fix codegen for CreateExternalRow with nested wide schema
## What changes were proposed in this pull request?
The wide schema, the expression of fields will be splitted into multiple functions, but the variable for loopVar can't be accessed in splitted functions, this PR change them as class member.
## How was this patch tested?
Added regression test.
Author: Davies Liu <da...@databricks.com>
Closes #12338 from davies/nested_row.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/372baf04
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/372baf04
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/372baf04
Branch: refs/heads/master
Commit: 372baf0479840695388515170e6eae0b3fc4125e
Parents: d187e7d
Author: Davies Liu <da...@databricks.com>
Authored: Tue Apr 12 17:26:37 2016 -0700
Committer: Davies Liu <da...@gmail.com>
Committed: Tue Apr 12 17:26:37 2016 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/objects.scala | 8 +++++---
.../sql/execution/datasources/json/JsonSuite.scala | 15 +++++++++++++++
2 files changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/372baf04/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
index 28b6b2a..26b1ff3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
@@ -446,6 +446,8 @@ case class MapObjects private(
override def genCode(ctx: CodegenContext, ev: ExprCode): String = {
val javaType = ctx.javaType(dataType)
val elementJavaType = ctx.javaType(loopVar.dataType)
+ ctx.addMutableState("boolean", loopVar.isNull, "")
+ ctx.addMutableState(elementJavaType, loopVar.value, "")
val genInputData = inputData.gen(ctx)
val genFunction = lambdaFunction.gen(ctx)
val dataLength = ctx.freshName("dataLength")
@@ -466,9 +468,9 @@ case class MapObjects private(
}
val loopNullCheck = if (primitiveElement) {
- s"boolean ${loopVar.isNull} = ${genInputData.value}.isNullAt($loopIndex);"
+ s"${loopVar.isNull} = ${genInputData.value}.isNullAt($loopIndex);"
} else {
- s"boolean ${loopVar.isNull} = ${genInputData.isNull} || ${loopVar.value} == null;"
+ s"${loopVar.isNull} = ${genInputData.isNull} || ${loopVar.value} == null;"
}
s"""
@@ -484,7 +486,7 @@ case class MapObjects private(
int $loopIndex = 0;
while ($loopIndex < $dataLength) {
- $elementJavaType ${loopVar.value} =
+ ${loopVar.value} =
($elementJavaType)${genInputData.value}${itemAccessor(loopIndex)};
$loopNullCheck
http://git-wip-us.apache.org/repos/asf/spark/blob/372baf04/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 2a18acb..e17340c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -1664,4 +1664,19 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
)
}
}
+
+ test("wide nested json table") {
+ val nested = (1 to 100).map { i =>
+ s"""
+ |"c$i": $i
+ """.stripMargin
+ }.mkString(", ")
+ val json = s"""
+ |{"a": [{$nested}], "b": [{$nested}]}
+ """.stripMargin
+ val rdd = sqlContext.sparkContext.makeRDD(Seq(json))
+ val df = sqlContext.read.json(rdd)
+ assert(df.schema.size === 2)
+ df.collect()
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org