You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/06/18 08:47:00 UTC
spark git commit: [SPARK-8283][SQL] Resolve udf_struct test failure
in HiveCompatibilitySuite
Repository: spark
Updated Branches:
refs/heads/master fee3438a3 -> e86fbdb1e
[SPARK-8283][SQL] Resolve udf_struct test failure in HiveCompatibilitySuite
This PR aimed to resolve udf_struct test failure in HiveCompatibilitySuite.
Currently, this is done by loosening CreateStruct's children type from NamedExpression to Expression and automatically generating StructField name for non-NamedExpression children.
The naming convention for unnamed children follows the udf's counterpart in Hive:
`col1, col2, col3, ...`
Author: Yijie Shen <he...@gmail.com>
Closes #6828 from yijieshen/SPARK-8283 and squashes the following commits:
6052b73 [Yijie Shen] Doc fix
677e0b7 [Yijie Shen] Resolve udf_struct test failure by automatically generate structField name for non-NamedExpression children
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e86fbdb1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e86fbdb1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e86fbdb1
Branch: refs/heads/master
Commit: e86fbdb1e6f1538f65ef78d90bbc41604f6bd580
Parents: fee3438
Author: Yijie Shen <he...@gmail.com>
Authored: Wed Jun 17 23:46:57 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed Jun 17 23:46:57 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/complexTypes.scala | 13 +++++++++----
.../sql/hive/execution/HiveCompatibilitySuite.scala | 2 +-
2 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e86fbdb1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
index 1aaf9b3..72fdceb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypes.scala
@@ -53,7 +53,7 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
* Returns a Row containing the evaluation of all children expressions.
* TODO: [[CreateStruct]] does not support codegen.
*/
-case class CreateStruct(children: Seq[NamedExpression]) extends Expression {
+case class CreateStruct(children: Seq[Expression]) extends Expression {
override def foldable: Boolean = children.forall(_.foldable)
@@ -62,9 +62,14 @@ case class CreateStruct(children: Seq[NamedExpression]) extends Expression {
override lazy val dataType: StructType = {
assert(resolved,
s"CreateStruct contains unresolvable children: ${children.filterNot(_.resolved)}.")
- val fields = children.map { child =>
- StructField(child.name, child.dataType, child.nullable, child.metadata)
- }
+ val fields = children.zipWithIndex.map { case (child, idx) =>
+ child match {
+ case ne: NamedExpression =>
+ StructField(ne.name, ne.dataType, ne.nullable, ne.metadata)
+ case _ =>
+ StructField(s"col${idx + 1}", child.dataType, child.nullable, Metadata.empty)
+ }
+ }
StructType(fields)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/e86fbdb1/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index 432de25..f88e627 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -933,7 +933,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"udf_stddev_pop",
"udf_stddev_samp",
"udf_string",
- // "udf_struct", TODO: FIX THIS and enable it.
+ "udf_struct",
"udf_substring",
"udf_subtract",
"udf_sum",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org