You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/01/30 00:45:03 UTC
spark git commit: [SPARK-5367][SQL] Support star expression in udf
Repository: spark
Updated Branches:
refs/heads/master de221ea03 -> fbaf9e089
[SPARK-5367][SQL] Support star expression in udf
now spark sql does not support star expression in udf, run the following sql by spark-sql will get error
```
select concat(*) from src
```
Author: wangfei <wa...@huawei.com>
Author: scwf <wa...@huawei.com>
Closes #4163 from scwf/udf-star and squashes the following commits:
9db7b39 [wangfei] addressed comments
da1da09 [scwf] minor fix
f87b5f9 [scwf] added test case
587bf7e [wangfei] compile fix
eb93c16 [wangfei] fix star resolve issue in udf
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fbaf9e08
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fbaf9e08
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fbaf9e08
Branch: refs/heads/master
Commit: fbaf9e08961551d3ae5c3629eca01e839b001b8e
Parents: de221ea
Author: wangfei <wa...@huawei.com>
Authored: Thu Jan 29 15:44:53 2015 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Thu Jan 29 15:44:53 2015 -0800
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 15 ++++++++++-----
.../spark/sql/hive/execution/HiveQuerySuite.scala | 5 +++++
2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fbaf9e08/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 7f4cc23..cefd70a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -250,6 +250,12 @@ class Analyzer(catalog: Catalog,
Project(
projectList.flatMap {
case s: Star => s.expand(child.output, resolver)
+ case Alias(f @ UnresolvedFunction(_, args), name) if containsStar(args) =>
+ val expandedArgs = args.flatMap {
+ case s: Star => s.expand(child.output, resolver)
+ case o => o :: Nil
+ }
+ Alias(child = f.copy(children = expandedArgs), name)() :: Nil
case o => o :: Nil
},
child)
@@ -273,10 +279,9 @@ class Analyzer(catalog: Catalog,
case q: LogicalPlan =>
logTrace(s"Attempting to resolve ${q.simpleString}")
q transformExpressions {
- case u @ UnresolvedAttribute(name)
- if resolver(name, VirtualColumn.groupingIdName) &&
- q.isInstanceOf[GroupingAnalytics] =>
- // Resolve the virtual column GROUPING__ID for the operator GroupingAnalytics
+ case u @ UnresolvedAttribute(name) if resolver(name, VirtualColumn.groupingIdName) &&
+ q.isInstanceOf[GroupingAnalytics] =>
+ // Resolve the virtual column GROUPING__ID for the operator GroupingAnalytics
q.asInstanceOf[GroupingAnalytics].gid
case u @ UnresolvedAttribute(name) =>
// Leave unchanged if resolution fails. Hopefully will be resolved next round.
@@ -299,7 +304,7 @@ class Analyzer(catalog: Catalog,
* Returns true if `exprs` contains a [[Star]].
*/
protected def containsStar(exprs: Seq[Expression]): Boolean =
- exprs.collect { case _: Star => true}.nonEmpty
+ exprs.exists(_.collect { case _: Star => true }.nonEmpty)
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/fbaf9e08/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 42819e3..60619f5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -509,6 +509,11 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
assert(sql("select key from src having key > 490").collect().size < 100)
}
+ test("SPARK-5367: resolve star expression in udf") {
+ assert(sql("select concat(*) from src limit 5").collect().size == 5)
+ assert(sql("select array(*) from src limit 5").collect().size == 5)
+ }
+
test("Query Hive native command execution result") {
val tableName = "test_native_commands"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org