You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2016/02/11 04:08:29 UTC
spark git commit: [SPARK-13205][SQL] SQL Generation Support for Self
Join
Repository: spark
Updated Branches:
refs/heads/master 663cc400f -> 0f09f0226
[SPARK-13205][SQL] SQL Generation Support for Self Join
This PR addresses two issues:
- Self join does not work in SQL Generation
- When creating new instances for `LogicalRelation`, `metastoreTableIdentifier` is lost.
liancheng Could you please review the code changes? Thank you!
Author: gatorsmile <ga...@gmail.com>
Closes #11084 from gatorsmile/selfJoinInSQLGen.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0f09f022
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0f09f022
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0f09f022
Branch: refs/heads/master
Commit: 0f09f0226983cdc409ef504dff48395787dc844f
Parents: 663cc40
Author: gatorsmile <ga...@gmail.com>
Authored: Thu Feb 11 11:08:21 2016 +0800
Committer: Cheng Lian <li...@databricks.com>
Committed: Thu Feb 11 11:08:21 2016 +0800
----------------------------------------------------------------------
.../spark/sql/execution/datasources/LogicalRelation.scala | 6 +++++-
.../main/scala/org/apache/spark/sql/hive/SQLBuilder.scala | 10 +++++++++-
.../org/apache/spark/sql/hive/LogicalPlanToSQLSuite.scala | 8 ++++++++
3 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0f09f022/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
index fa97f3d..0e0748f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/LogicalRelation.scala
@@ -76,7 +76,11 @@ case class LogicalRelation(
/** Used to lookup original attribute capitalization */
val attributeMap: AttributeMap[AttributeReference] = AttributeMap(output.map(o => (o, o)))
- def newInstance(): this.type = LogicalRelation(relation).asInstanceOf[this.type]
+ def newInstance(): this.type =
+ LogicalRelation(
+ relation,
+ expectedOutputAttributes,
+ metastoreTableIdentifier).asInstanceOf[this.type]
override def simpleString: String = s"Relation[${output.mkString(",")}] $relation"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/0f09f022/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
index fc5725d..4b75e60 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
@@ -142,7 +142,15 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
Some(s"`$database`.`$table`")
case Subquery(alias, child) =>
- toSQL(child).map(childSQL => s"($childSQL) AS $alias")
+ toSQL(child).map( childSQL =>
+ child match {
+ // Parentheses is not used for persisted data source relations
+ // e.g., select x.c1 from (t1) as x inner join (t1) as y on x.c1 = y.c1
+ case Subquery(_, _: LogicalRelation | _: MetastoreRelation) =>
+ s"$childSQL AS $alias"
+ case _ =>
+ s"($childSQL) AS $alias"
+ })
case Join(left, right, joinType, condition) =>
for {
http://git-wip-us.apache.org/repos/asf/spark/blob/0f09f022/sql/hive/src/test/scala/org/apache/spark/sql/hive/LogicalPlanToSQLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/LogicalPlanToSQLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/LogicalPlanToSQLSuite.scala
index 129bfe0..80ae312 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/LogicalPlanToSQLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/LogicalPlanToSQLSuite.scala
@@ -104,6 +104,14 @@ class LogicalPlanToSQLSuite extends SQLBuilderTest with SQLTestUtils {
checkHiveQl("SELECT id FROM t0 UNION ALL SELECT CAST(id AS INT) AS id FROM t0")
}
+ test("self join") {
+ checkHiveQl("SELECT x.key FROM t1 x JOIN t1 y ON x.key = y.key")
+ }
+
+ test("self join with group by") {
+ checkHiveQl("SELECT x.key, COUNT(*) FROM t1 x JOIN t1 y ON x.key = y.key group by x.key")
+ }
+
test("three-child union") {
checkHiveQl("SELECT id FROM t0 UNION ALL SELECT id FROM t0 UNION ALL SELECT id FROM t0")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org