You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/09/10 19:46:01 UTC
git commit: [SPARK-3362][SQL] Fix resolution for casewhen with nulls.
Repository: spark
Updated Branches:
refs/heads/master 6f7a76838 -> a0283300c
[SPARK-3362][SQL] Fix resolution for casewhen with nulls.
Current implementation will ignore else val type.
Author: Daoyuan Wang <da...@intel.com>
Closes #2245 from adrian-wang/casewhenbug and squashes the following commits:
3332f6e [Daoyuan Wang] remove wrong comment
83b536c [Daoyuan Wang] a comment to trigger retest
d7315b3 [Daoyuan Wang] code improve
eed35fc [Daoyuan Wang] bug in casewhen resolve
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a0283300
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a0283300
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a0283300
Branch: refs/heads/master
Commit: a0283300c4af5e64a1dc06193245daa1e746b5f4
Parents: 6f7a768
Author: Daoyuan Wang <da...@intel.com>
Authored: Wed Sep 10 10:45:15 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Sep 10 10:45:24 2014 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/expressions/predicates.scala | 5 +++--
...en then 1 else null end -0-f7c7fdd35c084bc797890aa08d33693c | 1 +
... then 1.0 else null end -0-aeb1f906bfe92f2d406f84109301afe0 | 1 +
...n then 1L else null end -0-763ae85e7a52b4cf4162d6a8931716bb | 1 +
...n then 1S else null end -0-6f5f3b3dbe9f1d1eb98443aef315b982 | 1 +
...n then 1Y else null end -0-589982a400d86157791c7216b10b6b5d | 1 +
...en then null else 1 end -0-48bd83660cf3ba93cdbdc24559092171 | 1 +
... then null else 1.0 end -0-7f5ce763801781cf568c6a31dd80b623 | 1 +
...n then null else 1L end -0-a7f1305ea4f86e596c368e35e45cc4e5 | 1 +
...n then null else 1S end -0-dfb61969e6cb6e6dbe89225b538c8d98 | 1 +
...n then null else 1Y end -0-7f4c32299c3738739b678ece62752a7b | 1 +
.../spark/sql/hive/execution/HiveTypeCoercionSuite.scala | 6 ++++++
12 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index 1313ccd..329af33 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -265,12 +265,13 @@ case class CaseWhen(branches: Seq[Expression]) extends Expression {
false
} else {
val allCondBooleans = predicates.forall(_.dataType == BooleanType)
- val dataTypesEqual = values.map(_.dataType).distinct.size <= 1
+ // both then and else val should be considered.
+ val dataTypesEqual = (values ++ elseValue).map(_.dataType).distinct.size <= 1
allCondBooleans && dataTypesEqual
}
}
- /** Written in imperative fashion for performance considerations. Same for CaseKeyWhen. */
+ /** Written in imperative fashion for performance considerations. */
override def eval(input: Row): Any = {
val len = branchesArr.length
var i = 0
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then 1 else null end -0-f7c7fdd35c084bc797890aa08d33693c
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then 1 else null end -0-f7c7fdd35c084bc797890aa08d33693c b/sql/hive/src/test/resources/golden/case when then 1 else null end -0-f7c7fdd35c084bc797890aa08d33693c
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then 1 else null end -0-f7c7fdd35c084bc797890aa08d33693c
@@ -0,0 +1 @@
+1
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then 1.0 else null end -0-aeb1f906bfe92f2d406f84109301afe0
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then 1.0 else null end -0-aeb1f906bfe92f2d406f84109301afe0 b/sql/hive/src/test/resources/golden/case when then 1.0 else null end -0-aeb1f906bfe92f2d406f84109301afe0
new file mode 100644
index 0000000..d3827e7
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then 1.0 else null end -0-aeb1f906bfe92f2d406f84109301afe0
@@ -0,0 +1 @@
+1.0
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then 1L else null end -0-763ae85e7a52b4cf4162d6a8931716bb
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then 1L else null end -0-763ae85e7a52b4cf4162d6a8931716bb b/sql/hive/src/test/resources/golden/case when then 1L else null end -0-763ae85e7a52b4cf4162d6a8931716bb
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then 1L else null end -0-763ae85e7a52b4cf4162d6a8931716bb
@@ -0,0 +1 @@
+1
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then 1S else null end -0-6f5f3b3dbe9f1d1eb98443aef315b982
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then 1S else null end -0-6f5f3b3dbe9f1d1eb98443aef315b982 b/sql/hive/src/test/resources/golden/case when then 1S else null end -0-6f5f3b3dbe9f1d1eb98443aef315b982
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then 1S else null end -0-6f5f3b3dbe9f1d1eb98443aef315b982
@@ -0,0 +1 @@
+1
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then 1Y else null end -0-589982a400d86157791c7216b10b6b5d
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then 1Y else null end -0-589982a400d86157791c7216b10b6b5d b/sql/hive/src/test/resources/golden/case when then 1Y else null end -0-589982a400d86157791c7216b10b6b5d
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then 1Y else null end -0-589982a400d86157791c7216b10b6b5d
@@ -0,0 +1 @@
+1
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then null else 1 end -0-48bd83660cf3ba93cdbdc24559092171
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then null else 1 end -0-48bd83660cf3ba93cdbdc24559092171 b/sql/hive/src/test/resources/golden/case when then null else 1 end -0-48bd83660cf3ba93cdbdc24559092171
new file mode 100644
index 0000000..7951def
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then null else 1 end -0-48bd83660cf3ba93cdbdc24559092171
@@ -0,0 +1 @@
+NULL
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then null else 1.0 end -0-7f5ce763801781cf568c6a31dd80b623
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then null else 1.0 end -0-7f5ce763801781cf568c6a31dd80b623 b/sql/hive/src/test/resources/golden/case when then null else 1.0 end -0-7f5ce763801781cf568c6a31dd80b623
new file mode 100644
index 0000000..7951def
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then null else 1.0 end -0-7f5ce763801781cf568c6a31dd80b623
@@ -0,0 +1 @@
+NULL
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then null else 1L end -0-a7f1305ea4f86e596c368e35e45cc4e5
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then null else 1L end -0-a7f1305ea4f86e596c368e35e45cc4e5 b/sql/hive/src/test/resources/golden/case when then null else 1L end -0-a7f1305ea4f86e596c368e35e45cc4e5
new file mode 100644
index 0000000..7951def
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then null else 1L end -0-a7f1305ea4f86e596c368e35e45cc4e5
@@ -0,0 +1 @@
+NULL
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then null else 1S end -0-dfb61969e6cb6e6dbe89225b538c8d98
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then null else 1S end -0-dfb61969e6cb6e6dbe89225b538c8d98 b/sql/hive/src/test/resources/golden/case when then null else 1S end -0-dfb61969e6cb6e6dbe89225b538c8d98
new file mode 100644
index 0000000..7951def
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then null else 1S end -0-dfb61969e6cb6e6dbe89225b538c8d98
@@ -0,0 +1 @@
+NULL
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/resources/golden/case when then null else 1Y end -0-7f4c32299c3738739b678ece62752a7b
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/case when then null else 1Y end -0-7f4c32299c3738739b678ece62752a7b b/sql/hive/src/test/resources/golden/case when then null else 1Y end -0-7f4c32299c3738739b678ece62752a7b
new file mode 100644
index 0000000..7951def
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/case when then null else 1Y end -0-7f4c32299c3738739b678ece62752a7b
@@ -0,0 +1 @@
+NULL
http://git-wip-us.apache.org/repos/asf/spark/blob/a0283300/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala
index c3c18cf..48fffe5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala
@@ -33,6 +33,12 @@ class HiveTypeCoercionSuite extends HiveComparisonTest {
}
}
+ val nullVal = "null"
+ baseTypes.init.foreach { i =>
+ createQueryTest(s"case when then $i else $nullVal end ", s"SELECT case when true then $i else $nullVal end FROM src limit 1")
+ createQueryTest(s"case when then $nullVal else $i end ", s"SELECT case when true then $nullVal else $i end FROM src limit 1")
+ }
+
test("[SPARK-2210] boolean cast on boolean value should be removed") {
val q = "select cast(cast(key=0 as boolean) as boolean) from src"
val project = TestHive.sql(q).queryExecution.executedPlan.collect { case e: Project => e }.head
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org