You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2016/10/14 21:45:23 UTC
spark git commit: [SPARK-17863][SQL] should not add column into
Distinct
Repository: spark
Updated Branches:
refs/heads/master 522dd0d0e -> da9aeb0fd
[SPARK-17863][SQL] should not add column into Distinct
## What changes were proposed in this pull request?
We are trying to resolve the attribute in sort by pulling up some column for grandchild into child, but that's wrong when the child is Distinct, because the added column will change the behavior of Distinct, we should not do that.
## How was this patch tested?
Added regression test.
Author: Davies Liu <da...@databricks.com>
Closes #15489 from davies/order_distinct.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/da9aeb0f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/da9aeb0f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/da9aeb0f
Branch: refs/heads/master
Commit: da9aeb0fde589f7c21c2f4a32036a68c0353965d
Parents: 522dd0d
Author: Davies Liu <da...@databricks.com>
Authored: Fri Oct 14 14:45:20 2016 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Fri Oct 14 14:45:20 2016 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 2 ++
.../org/apache/spark/sql/SQLQuerySuite.scala | 24 ++++++++++++++++++++
2 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/da9aeb0f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 536d387..f8f4799 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -838,6 +838,8 @@ class Analyzer(
// attributes that its child might have or could have.
val missing = missingAttrs -- g.child.outputSet
g.copy(join = true, child = addMissingAttr(g.child, missing))
+ case d: Distinct =>
+ throw new AnalysisException(s"Can't add $missingAttrs to $d")
case u: UnaryNode =>
u.withNewChildren(addMissingAttr(u.child, missingAttrs) :: Nil)
case other =>
http://git-wip-us.apache.org/repos/asf/spark/blob/da9aeb0f/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 0ee8c95..60978ef 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -1106,6 +1106,30 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
)
}
+ test("SPARK-17863: SELECT distinct does not work correctly if order by missing attribute") {
+ checkAnswer(
+ sql("""select distinct struct.a, struct.b
+ |from (
+ | select named_struct('a', 1, 'b', 2, 'c', 3) as struct
+ | union all
+ | select named_struct('a', 1, 'b', 2, 'c', 4) as struct) tmp
+ |order by a, b
+ |""".stripMargin),
+ Row(1, 2) :: Nil)
+
+ val error = intercept[AnalysisException] {
+ sql("""select distinct struct.a, struct.b
+ |from (
+ | select named_struct('a', 1, 'b', 2, 'c', 3) as struct
+ | union all
+ | select named_struct('a', 1, 'b', 2, 'c', 4) as struct) tmp
+ |order by struct.a, struct.b
+ |""".stripMargin)
+ }
+ assert(error.message contains "cannot resolve '`struct.a`' given input columns: [a, b]")
+
+ }
+
test("cast boolean to string") {
// TODO Ensure true/false string letter casing is consistent with Hive in all cases.
checkAnswer(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org