You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/10/12 18:16:18 UTC
spark git commit: [SPARK-10960] [SQL] SQL with windowing function
should be able to refer column in inner select
Repository: spark
Updated Branches:
refs/heads/master 595012ea8 -> fcb37a041
[SPARK-10960] [SQL] SQL with windowing function should be able to refer column in inner select
JIRA: https://issues.apache.org/jira/browse/SPARK-10960
When accessing a column in inner select from a select with window function, `AnalysisException` will be thrown. For example, an query like this:
select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1 from (select month, area, product, 1 as tmp1 from windowData) tmp
Currently, the rule `ExtractWindowExpressions` in `Analyzer` only extracts regular expressions from `WindowFunction`, `WindowSpecDefinition` and `AggregateExpression`. We need to also extract other attributes as the one in `Alias` as shown in the above query.
Author: Liang-Chi Hsieh <vi...@appier.com>
Closes #9011 from viirya/fix-window-inner-column.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fcb37a04
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fcb37a04
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fcb37a04
Branch: refs/heads/master
Commit: fcb37a04177edc2376e39dd0b910f0268f7c72ec
Parents: 595012e
Author: Liang-Chi Hsieh <vi...@appier.com>
Authored: Mon Oct 12 09:16:14 2015 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Mon Oct 12 09:16:14 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 4 +++
.../sql/hive/execution/SQLQuerySuite.scala | 27 ++++++++++++++++++++
2 files changed, 31 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fcb37a04/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index bf72d47..f5597a0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -831,6 +831,10 @@ class Analyzer(
val withName = Alias(agg, s"_w${extractedExprBuffer.length}")()
extractedExprBuffer += withName
withName.toAttribute
+
+ // Extracts other attributes
+ case attr: Attribute => extractExpr(attr)
+
}.asInstanceOf[NamedExpression]
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fcb37a04/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index ccc15ea..51b63f3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -838,6 +838,33 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
).map(i => Row(i._1, i._2, i._3)))
}
+ test("window function: refer column in inner select block") {
+ val data = Seq(
+ WindowData(1, "a", 5),
+ WindowData(2, "a", 6),
+ WindowData(3, "b", 7),
+ WindowData(4, "b", 8),
+ WindowData(5, "c", 9),
+ WindowData(6, "c", 10)
+ )
+ sparkContext.parallelize(data).toDF().registerTempTable("windowData")
+
+ checkAnswer(
+ sql(
+ """
+ |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
+ |from (select month, area, product, 1 as tmp1 from windowData) tmp
+ """.stripMargin),
+ Seq(
+ ("a", 2),
+ ("a", 3),
+ ("b", 2),
+ ("b", 3),
+ ("c", 2),
+ ("c", 3)
+ ).map(i => Row(i._1, i._2)))
+ }
+
test("window function: partition and order expressions") {
val data = Seq(
WindowData(1, "a", 5),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org