You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/12/23 00:21:54 UTC
spark git commit: [SPARK-12102][SQL] Cast a non-nullable struct field
to a nullable field during analysis
Repository: spark
Updated Branches:
refs/heads/master 575a13279 -> b374a2583
[SPARK-12102][SQL] Cast a non-nullable struct field to a nullable field during analysis
Compare both left and right side of the case expression ignoring nullablity when checking for type equality.
Author: Dilip Biswal <db...@us.ibm.com>
Closes #10156 from dilipbiswal/spark-12102.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b374a258
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b374a258
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b374a258
Branch: refs/heads/master
Commit: b374a25831af031f461716c52b615665aa5392c2
Parents: 575a132
Author: Dilip Biswal <db...@us.ibm.com>
Authored: Tue Dec 22 15:21:49 2015 -0800
Committer: Yin Huai <yh...@databricks.com>
Committed: Tue Dec 22 15:21:49 2015 -0800
----------------------------------------------------------------------
.../sql/catalyst/expressions/conditionalExpressions.scala | 4 +++-
.../org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala | 6 ++++++
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b374a258/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
index 40b1eec..f79c867 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala
@@ -91,7 +91,9 @@ trait CaseWhenLike extends Expression {
// both then and else expressions should be considered.
def valueTypes: Seq[DataType] = (thenList ++ elseValue).map(_.dataType)
- def valueTypesEqual: Boolean = valueTypes.distinct.size == 1
+ def valueTypesEqual: Boolean = valueTypes.size <= 1 || valueTypes.sliding(2, 1).forall {
+ case Seq(dt1, dt2) => dt1.sameType(dt2)
+ }
override def checkInputDataTypes(): TypeCheckResult = {
if (valueTypesEqual) {
http://git-wip-us.apache.org/repos/asf/spark/blob/b374a258/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index aeeca80..fa823e3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -274,4 +274,10 @@ class AnalysisSuite extends AnalysisTest {
assert(lits(1) >= min && lits(1) <= max)
assert(lits(0) == lits(1))
}
+
+ test("SPARK-12102: Ignore nullablity when comparing two sides of case") {
+ val relation = LocalRelation('a.struct('x.int), 'b.struct('x.int.withNullability(false)))
+ val plan = relation.select(CaseWhen(Seq(Literal(true), 'a, 'b)).as("val"))
+ assertAnalysisSuccess(plan)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org