You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/07/08 23:29:58 UTC
spark git commit: [SPARK-8783] [SQL] CTAS with WITH clause does not
work
Repository: spark
Updated Branches:
refs/heads/master 2b40365d7 -> f03154378
[SPARK-8783] [SQL] CTAS with WITH clause does not work
Currently, CTESubstitution only handles the case that WITH is on the top of the plan.
I think it SHOULD handle the case that WITH is child of CTAS.
This patch simply changes 'match' to 'transform' for recursive search of WITH in the plan.
Author: Keuntae Park <si...@apache.org>
Closes #7180 from sirpkt/SPARK-8783 and squashes the following commits:
e4428f0 [Keuntae Park] Merge remote-tracking branch 'upstream/master' into CTASwithWITH
1671c77 [Keuntae Park] WITH clause can be inside CTAS
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f0315437
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f0315437
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f0315437
Branch: refs/heads/master
Commit: f031543782e8f0f5b6a4471ba1c1d5c53efbe5cd
Parents: 2b40365
Author: Keuntae Park <si...@apache.org>
Authored: Wed Jul 8 14:29:52 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Jul 8 14:29:52 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../spark/sql/hive/execution/SQLQuerySuite.scala | 18 ++++++++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f0315437/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 15e84e6..3fdc6d6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -85,7 +85,7 @@ class Analyzer(
*/
object CTESubstitution extends Rule[LogicalPlan] {
// TODO allow subquery to define CTE
- def apply(plan: LogicalPlan): LogicalPlan = plan match {
+ def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case With(child, relations) => substituteCTE(child, relations)
case other => other
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f0315437/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index bf9f2ec..05a1f00 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -159,6 +159,24 @@ class SQLQuerySuite extends QueryTest {
checkAnswer(query, Row(1, 1) :: Nil)
}
+ test("CTAS with WITH clause") {
+ val df = Seq((1, 1)).toDF("c1", "c2")
+ df.registerTempTable("table1")
+
+ sql(
+ """
+ |CREATE TABLE with_table1 AS
+ |WITH T AS (
+ | SELECT *
+ | FROM table1
+ |)
+ |SELECT *
+ |FROM T
+ """.stripMargin)
+ val query = sql("SELECT * FROM with_table1")
+ checkAnswer(query, Row(1, 1) :: Nil)
+ }
+
test("explode nested Field") {
Seq(NestedArray1(NestedArray2(Seq(1, 2, 3)))).toDF.registerTempTable("nestedArray")
checkAnswer(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org