You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/04/12 16:08:41 UTC
spark git commit: [SPARK-20291][SQL][BACKPORT] NaNvl(FloatType, NullType) should not be cast to N…
Repository: spark
Updated Branches:
refs/heads/branch-2.0 123a758ce -> 24f6ef25a
[SPARK-20291][SQL][BACKPORT] NaNvl(FloatType, NullType) should not be cast to N\u2026
\u2026aNvl(DoubleType, DoubleType)
## What changes were proposed in this pull request?
This is a backport of https://github.com/apache/spark/pull/17606
`NaNvl(float value, null)` will be converted into `NaNvl(float value, Cast(null, DoubleType))` and finally `NaNvl(Cast(float value, DoubleType), Cast(null, DoubleType))`.
This will cause mismatching in the output type when the input type is float.
By adding extra rule in TypeCoercion can resolve this issue.
## How was this patch tested?
unite tests.
Please review http://spark.apache.org/contributing.html before opening a pull request.
Author: DB Tsai <db...@netflix.com>
Author: DB Tsai <db...@dbtsai.com>
Closes #17618 from dbtsai/branch-2.0.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/24f6ef25
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/24f6ef25
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/24f6ef25
Branch: refs/heads/branch-2.0
Commit: 24f6ef25ab042fac49d151d889a167bc254ebd43
Parents: 123a758
Author: DB Tsai <db...@netflix.com>
Authored: Wed Apr 12 09:08:37 2017 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Wed Apr 12 09:08:37 2017 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/TypeCoercion.scala | 1 +
.../sql/catalyst/analysis/TypeCoercionSuite.scala | 14 ++++++++++----
.../org/apache/spark/sql/DataFrameNaFunctions.scala | 3 +--
3 files changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/24f6ef25/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index 7e84dc8..deb6063 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -509,6 +509,7 @@ object TypeCoercion {
NaNvl(l, Cast(r, DoubleType))
case NaNvl(l, r) if l.dataType == FloatType && r.dataType == DoubleType =>
NaNvl(Cast(l, DoubleType), r)
+ case NaNvl(l, r) if r.dataType == NullType => NaNvl(l, Cast(r, l.dataType))
case e: RuntimeReplaceable => e.replaceForTypeCoercion()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/24f6ef25/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index 9560563..0fda97a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -416,14 +416,20 @@ class TypeCoercionSuite extends PlanTest {
test("nanvl casts") {
ruleTest(TypeCoercion.FunctionArgumentConversion,
- NaNvl(Literal.create(1.0, FloatType), Literal.create(1.0, DoubleType)),
- NaNvl(Cast(Literal.create(1.0, FloatType), DoubleType), Literal.create(1.0, DoubleType)))
+ NaNvl(Literal.create(1.0f, FloatType), Literal.create(1.0, DoubleType)),
+ NaNvl(Cast(Literal.create(1.0f, FloatType), DoubleType), Literal.create(1.0, DoubleType)))
ruleTest(TypeCoercion.FunctionArgumentConversion,
- NaNvl(Literal.create(1.0, DoubleType), Literal.create(1.0, FloatType)),
- NaNvl(Literal.create(1.0, DoubleType), Cast(Literal.create(1.0, FloatType), DoubleType)))
+ NaNvl(Literal.create(1.0, DoubleType), Literal.create(1.0f, FloatType)),
+ NaNvl(Literal.create(1.0, DoubleType), Cast(Literal.create(1.0f, FloatType), DoubleType)))
ruleTest(TypeCoercion.FunctionArgumentConversion,
NaNvl(Literal.create(1.0, DoubleType), Literal.create(1.0, DoubleType)),
NaNvl(Literal.create(1.0, DoubleType), Literal.create(1.0, DoubleType)))
+ ruleTest(TypeCoercion.FunctionArgumentConversion,
+ NaNvl(Literal.create(1.0f, FloatType), Literal.create(null, NullType)),
+ NaNvl(Literal.create(1.0f, FloatType), Cast(Literal.create(null, NullType), FloatType)))
+ ruleTest(TypeCoercion.FunctionArgumentConversion,
+ NaNvl(Literal.create(1.0, DoubleType), Literal.create(null, NullType)),
+ NaNvl(Literal.create(1.0, DoubleType), Cast(Literal.create(null, NullType), DoubleType)))
}
test("type coercion for If") {
http://git-wip-us.apache.org/repos/asf/spark/blob/24f6ef25/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index f37dece..8e4e71f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -408,8 +408,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
val quotedColName = "`" + col.name + "`"
val colValue = col.dataType match {
case DoubleType | FloatType =>
- // nanvl only supports these types
- nanvl(df.col(quotedColName), lit(null).cast(col.dataType))
+ nanvl(df.col(quotedColName), lit(null)) // nanvl only supports these types
case _ => df.col(quotedColName)
}
coalesce(colValue, lit(replacement).cast(col.dataType)).as(col.name)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org