You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2016/02/19 21:22:27 UTC
spark git commit: [SPARK-13384][SQL] Keep attribute qualifiers after
dedup in Analyzer
Repository: spark
Updated Branches:
refs/heads/master 6915cc23b -> c7c55637b
[SPARK-13384][SQL] Keep attribute qualifiers after dedup in Analyzer
JIRA: https://issues.apache.org/jira/browse/SPARK-13384
## What changes were proposed in this pull request?
When we de-duplicate attributes in Analyzer, we create new attributes. However, we don't keep original qualifiers. Some plans will be failed to analysed. We should keep original qualifiers in new attributes.
## How was the this patch tested?
Unit test is added.
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #11261 from viirya/keep-attr-qualifiers.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c7c55637
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c7c55637
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c7c55637
Branch: refs/heads/master
Commit: c7c55637bfc523237f5cc5c5b61837b1e3d5fdfc
Parents: 6915cc2
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Fri Feb 19 12:22:22 2016 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Fri Feb 19 12:22:22 2016 -0800
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/analysis/Analyzer.scala | 3 ++-
.../spark/sql/catalyst/analysis/AnalysisSuite.scala | 14 ++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c7c55637/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 004c1ca..8368657 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -432,7 +432,8 @@ class Analyzer(
case r if r == oldRelation => newRelation
} transformUp {
case other => other transformExpressions {
- case a: Attribute => attributeRewrites.get(a).getOrElse(a)
+ case a: Attribute =>
+ attributeRewrites.get(a).getOrElse(a).withQualifiers(a.qualifiers)
}
}
newRight
http://git-wip-us.apache.org/repos/asf/spark/blob/c7c55637/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index f85ae24..92db029 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.Inner
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.types._
@@ -336,4 +337,17 @@ class AnalysisSuite extends AnalysisTest {
val plan = relation.select(CaseWhen(Seq((Literal(true), 'a.attr)), 'b).as("val"))
assertAnalysisSuccess(plan)
}
+
+ test("Keep attribute qualifiers after dedup") {
+ val input = LocalRelation('key.int, 'value.string)
+
+ val query =
+ Project(Seq($"x.key", $"y.key"),
+ Join(
+ Project(Seq($"x.key"), Subquery("x", input)),
+ Project(Seq($"y.key"), Subquery("y", input)),
+ Inner, None))
+
+ assertAnalysisSuccess(query)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org