You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by da...@apache.org on 2016/06/29 21:43:02 UTC
spark git commit: [SPARK-16301] [SQL] The analyzer rule for resolving
using joins should respect the case sensitivity setting.
Repository: spark
Updated Branches:
refs/heads/master d8a87a3ed -> 8b5a8b25b
[SPARK-16301] [SQL] The analyzer rule for resolving using joins should respect the case sensitivity setting.
## What changes were proposed in this pull request?
The analyzer rule for resolving using joins should respect the case sensitivity setting.
## How was this patch tested?
New tests in ResolveNaturalJoinSuite
Author: Yin Huai <yh...@databricks.com>
Closes #13977 from yhuai/SPARK-16301.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8b5a8b25
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8b5a8b25
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8b5a8b25
Branch: refs/heads/master
Commit: 8b5a8b25b9d29b7d0949d5663c7394b26154a836
Parents: d8a87a3
Author: Yin Huai <yh...@databricks.com>
Authored: Wed Jun 29 14:42:58 2016 -0700
Committer: Davies Liu <da...@gmail.com>
Committed: Wed Jun 29 14:42:58 2016 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 26 ++++++++++++-----
.../analysis/ResolveNaturalJoinSuite.scala | 30 ++++++++++++++++++++
2 files changed, 49 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8b5a8b25/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 96f2e38..d1d2c59 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1836,13 +1836,25 @@ class Analyzer(
}
private def commonNaturalJoinProcessing(
- left: LogicalPlan,
- right: LogicalPlan,
- joinType: JoinType,
- joinNames: Seq[String],
- condition: Option[Expression]) = {
- val leftKeys = joinNames.map(keyName => left.output.find(_.name == keyName).get)
- val rightKeys = joinNames.map(keyName => right.output.find(_.name == keyName).get)
+ left: LogicalPlan,
+ right: LogicalPlan,
+ joinType: JoinType,
+ joinNames: Seq[String],
+ condition: Option[Expression]) = {
+ val leftKeys = joinNames.map { keyName =>
+ val joinColumn = left.output.find(attr => resolver(attr.name, keyName))
+ assert(
+ joinColumn.isDefined,
+ s"$keyName should exist in ${left.output.map(_.name).mkString(",")}")
+ joinColumn.get
+ }
+ val rightKeys = joinNames.map { keyName =>
+ val joinColumn = right.output.find(attr => resolver(attr.name, keyName))
+ assert(
+ joinColumn.isDefined,
+ s"$keyName should exist in ${right.output.map(_.name).mkString(",")}")
+ joinColumn.get
+ }
val joinPairs = leftKeys.zip(rightKeys)
val newCondition = (condition ++ joinPairs.map(EqualTo.tupled)).reduceOption(And)
http://git-wip-us.apache.org/repos/asf/spark/blob/8b5a8b25/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
index 748579d..100ec4d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveNaturalJoinSuite.scala
@@ -113,4 +113,34 @@ class ResolveNaturalJoinSuite extends AnalysisTest {
assert(error.message.contains(
"using columns ['d] can not be resolved given input columns: [b, a, c]"))
}
+
+ test("using join with a case sensitive analyzer") {
+ val expected = r1.join(r2, Inner, Some(EqualTo(a, a))).select(a, b, c)
+
+ {
+ val usingPlan = r1.join(r2, UsingJoin(Inner, Seq(UnresolvedAttribute("a"))), None)
+ checkAnalysis(usingPlan, expected, caseSensitive = true)
+ }
+
+ {
+ val usingPlan = r1.join(r2, UsingJoin(Inner, Seq(UnresolvedAttribute("A"))), None)
+ assertAnalysisError(
+ usingPlan,
+ Seq("using columns ['A] can not be resolved given input columns: [b, a, c, a]"))
+ }
+ }
+
+ test("using join with a case insensitive analyzer") {
+ val expected = r1.join(r2, Inner, Some(EqualTo(a, a))).select(a, b, c)
+
+ {
+ val usingPlan = r1.join(r2, UsingJoin(Inner, Seq(UnresolvedAttribute("a"))), None)
+ checkAnalysis(usingPlan, expected, caseSensitive = false)
+ }
+
+ {
+ val usingPlan = r1.join(r2, UsingJoin(Inner, Seq(UnresolvedAttribute("A"))), None)
+ checkAnalysis(usingPlan, expected, caseSensitive = false)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org