You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2021/04/21 07:23:21 UTC
[spark] branch branch-2.4 updated: [SPARK-31225][SQL][2.4] Override
sql method of OuterReference
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 1630d64 [SPARK-31225][SQL][2.4] Override sql method of OuterReference
1630d64 is described below
commit 1630d64cab216f1404bf0940483ec3ecb86732d1
Author: Kent Yao <ya...@apache.org>
AuthorDate: Wed Apr 21 07:22:51 2021 +0000
[SPARK-31225][SQL][2.4] Override sql method of OuterReference
### What changes were proposed in this pull request?
Per https://github.com/apache/spark/pull/32179#issuecomment-822954063 's request to backport SPARK-31225 to branch 2.4
### Why are the changes needed?
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
Closes #32256 from yaooqinn/SPARK-31225.
Authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../sql/catalyst/expressions/namedExpressions.scala | 1 +
.../sql/catalyst/analysis/AnalysisErrorSuite.scala | 12 ++++++------
.../test/resources/sql-tests/results/having.sql.out | 18 +++++++++---------
.../negative-cases/invalid-correlation.sql.out | 2 +-
.../scala/org/apache/spark/sql/SubquerySuite.scala | 2 +-
5 files changed, 18 insertions(+), 17 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 049ea77..76d0ed0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -378,6 +378,7 @@ case class OuterReference(e: NamedExpression)
override def nullable: Boolean = e.nullable
override def prettyName: String = "outer"
+ override def sql: String = s"$prettyName(${e.sql})"
override def name: String = e.name
override def qualifier: Seq[String] = e.qualifier
override def exprId: ExprId = e.exprId
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 41c3c88..957c1b7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -641,12 +641,12 @@ class AnalysisErrorSuite extends AnalysisTest {
val t1 = LocalRelation(a, b)
val t2 = LocalRelation(c)
val conditions = Seq(
- (abs($"a") === $"c", "abs(`a`) = outer()"),
- (abs($"a") <=> $"c", "abs(`a`) <=> outer()"),
- ($"a" + 1 === $"c", "(`a` + 1) = outer()"),
- ($"a" + $"b" === $"c", "(`a` + `b`) = outer()"),
- ($"a" + $"c" === $"b", "(`a` + outer()) = `b`"),
- (And($"a" === $"c", Cast($"a", IntegerType) === $"c"), "CAST(`a` AS INT) = outer()"))
+ (abs($"a") === $"c", "abs(`a`) = outer(`c`)"),
+ (abs($"a") <=> $"c", "abs(`a`) <=> outer(`c`)"),
+ ($"a" + 1 === $"c", "(`a` + 1) = outer(`c`)"),
+ ($"a" + $"b" === $"c", "(`a` + `b`) = outer(`c`)"),
+ ($"a" + $"c" === $"b", "(`a` + outer(`c`)) = `b`"),
+ (And($"a" === $"c", Cast($"a", IntegerType) === $"c"), "CAST(`a` AS INT) = outer(`c`)"))
conditions.foreach { case (cond, msg) =>
val plan = Project(
ScalarSubquery(
diff --git a/sql/core/src/test/resources/sql-tests/results/having.sql.out b/sql/core/src/test/resources/sql-tests/results/having.sql.out
index ee1956e..2a3e68a 100644
--- a/sql/core/src/test/resources/sql-tests/results/having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/having.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 9
+-- Number of queries: 13
-- !query 0
@@ -85,31 +85,31 @@ struct<b:bigint>
-- !query 9
SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY GROUPING SETS(t.c1) HAVING t.c1 = 1
--- !query schema
+-- !query 9 schema
struct<c1:int>
--- !query output
+-- !query 9 output
1
-- !query 10
SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY CUBE(t.c1) HAVING t.c1 = 1
--- !query schema
+-- !query 10 schema
struct<c1:int>
--- !query output
+-- !query 10 output
1
-- !query 11
SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY ROLLUP(t.c1) HAVING t.c1 = 1
--- !query schema
+-- !query 11 schema
struct<c1:int>
--- !query output
+-- !query 11 output
1
-- !query 12
SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY t.c1 HAVING t.c1 = 1
--- !query schema
+-- !query 12 schema
struct<c1:int>
--- !query output
+-- !query 12 output
1
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
index e49978d..11157f0 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
@@ -108,7 +108,7 @@ struct<>
-- !query 7 output
org.apache.spark.sql.AnalysisException
Expressions referencing the outer query are not supported outside of WHERE/HAVING clauses:
-Aggregate [min(outer(t2a#x)) AS min(outer())#x]
+Aggregate [min(outer(t2a#x)) AS min(outer(t2.`t2a`))#x]
+- SubqueryAlias `t3`
+- Project [t3a#x, t3b#x, t3c#x]
+- SubqueryAlias `t3`
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
index 5fb992e..bfd884f1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -536,7 +536,7 @@ class SubquerySuite extends QueryTest with SharedSQLContext {
sql("select a, (select sum(b) from l l2 where l2.a < l1.a) sum_b from l l1")
}
assert(msg1.getMessage.contains(
- "Correlated column is not allowed in predicate (l2.`a` < outer())"))
+ "Correlated column is not allowed in predicate (l2.`a` < outer(l1.`a`))"))
}
test("disjunctive correlated scalar subquery") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org