You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/10/21 23:29:27 UTC
spark git commit: [SPARK-8654][SQL] Analysis exception when using
NULL IN (...) : invalid cast
Repository: spark
Updated Branches:
refs/heads/master 19ad18638 -> dce2f8c9d
[SPARK-8654][SQL] Analysis exception when using NULL IN (...) : invalid cast
In the analysis phase , while processing the rules for IN predicate, we
compare the in-list types to the lhs expression type and generate
cast operation if necessary. In the case of NULL [NOT] IN expr1 , we end up
generating cast between in list types to NULL like cast (1 as NULL) which
is not a valid cast.
The fix is to find a common type between LHS and RHS expressions and cast
all the expression to the common type.
Author: Dilip Biswal <db...@us.ibm.com>
This patch had conflicts when merged, resolved by
Committer: Michael Armbrust <mi...@databricks.com>
Closes #9036 from dilipbiswal/spark_8654_new.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dce2f8c9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dce2f8c9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dce2f8c9
Branch: refs/heads/master
Commit: dce2f8c9d74bda46f3c7a7ebe7372b04d6b17a61
Parents: 19ad186
Author: Dilip Biswal <db...@us.ibm.com>
Authored: Wed Oct 21 14:29:03 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Oct 21 14:29:03 2015 -0700
----------------------------------------------------------------------
.../catalyst/analysis/HiveTypeCoercion.scala | 11 ++++++++--
.../sql/catalyst/analysis/AnalysisSuite.scala | 21 ++++++++++++++++++++
.../analysis/HiveTypeCoercionSuite.scala | 3 ++-
3 files changed, 32 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/dce2f8c9/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
index 87a3845..1140150 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
@@ -304,7 +304,11 @@ object HiveTypeCoercion {
}
/**
- * Convert all expressions in in() list to the left operator type
+ * Convert the value and in list expressions to the common operator type
+ * by looking at all the argument types and finding the closest one that
+ * all the arguments can be cast to. When no common operator type is found
+ * the original expression will be returned and an Analysis Exception will
+ * be raised at type checking phase.
*/
object InConversion extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
@@ -312,7 +316,10 @@ object HiveTypeCoercion {
case e if !e.childrenResolved => e
case i @ In(a, b) if b.exists(_.dataType != a.dataType) =>
- i.makeCopy(Array(a, b.map(Cast(_, a.dataType))))
+ findWiderCommonType(i.children.map(_.dataType)) match {
+ case Some(finalDataType) => i.withNewChildren(i.children.map(Cast(_, finalDataType)))
+ case None => i
+ }
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/dce2f8c9/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 0a1fa74..71d2939 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -153,4 +153,25 @@ class AnalysisSuite extends AnalysisTest {
checkAnalysis(plan, expected)
}
+
+ test("SPARK-8654: invalid CAST in NULL IN(...) expression") {
+ val plan = Project(Alias(In(Literal(null), Seq(Literal(1), Literal(2))), "a")() :: Nil,
+ LocalRelation()
+ )
+ assertAnalysisSuccess(plan)
+ }
+
+ test("SPARK-8654: different types in inlist but can be converted to a commmon type") {
+ val plan = Project(Alias(In(Literal(null), Seq(Literal(1), Literal(1.2345))), "a")() :: Nil,
+ LocalRelation()
+ )
+ assertAnalysisSuccess(plan)
+ }
+
+ test("SPARK-8654: check type compatibility error") {
+ val plan = Project(Alias(In(Literal(null), Seq(Literal(true), Literal(1))), "a")() :: Nil,
+ LocalRelation()
+ )
+ assertAnalysisError(plan, Seq("data type mismatch: Arguments must be same type"))
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/dce2f8c9/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
index 6f33ab7..d3fafaa 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
@@ -470,7 +470,8 @@ class HiveTypeCoercionSuite extends PlanTest {
)
ruleTest(inConversion,
In(Literal("a"), Seq(Literal(1), Literal("b"))),
- In(Literal("a"), Seq(Cast(Literal(1), StringType), Cast(Literal("b"), StringType)))
+ In(Cast(Literal("a"), StringType),
+ Seq(Cast(Literal(1), StringType), Cast(Literal("b"), StringType)))
)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org