You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/11/14 23:57:01 UTC
spark git commit: [SPARK-4390][SQL] Handle NaN cast to decimal
correctly
Repository: spark
Updated Branches:
refs/heads/master 5930f64bf -> a0300ea32
[SPARK-4390][SQL] Handle NaN cast to decimal correctly
Author: Michael Armbrust <mi...@databricks.com>
Closes #3256 from marmbrus/NanDecimal and squashes the following commits:
4c3ba46 [Michael Armbrust] fix style
d360f83 [Michael Armbrust] Handle NaN cast to decimal
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a0300ea3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a0300ea3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a0300ea3
Branch: refs/heads/master
Commit: a0300ea32a9d92bd51c72930bc3979087b0082b2
Parents: 5930f64
Author: Michael Armbrust <mi...@databricks.com>
Authored: Fri Nov 14 14:56:57 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Fri Nov 14 14:56:57 2014 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/catalyst/expressions/Cast.scala | 6 +++++-
.../golden/NaN to Decimal-0-6ca781bc343025635d72321ef0a9d425 | 1 +
.../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 3 +++
3 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a0300ea3/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 55319e7..34697a1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -290,7 +290,11 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
case LongType =>
b => changePrecision(Decimal(b.asInstanceOf[Long]), target)
case x: NumericType => // All other numeric types can be represented precisely as Doubles
- b => changePrecision(Decimal(x.numeric.asInstanceOf[Numeric[Any]].toDouble(b)), target)
+ b => try {
+ changePrecision(Decimal(x.numeric.asInstanceOf[Numeric[Any]].toDouble(b)), target)
+ } catch {
+ case _: NumberFormatException => null
+ }
}
// DoubleConverter
http://git-wip-us.apache.org/repos/asf/spark/blob/a0300ea3/sql/hive/src/test/resources/golden/NaN to Decimal-0-6ca781bc343025635d72321ef0a9d425
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/NaN to Decimal-0-6ca781bc343025635d72321ef0a9d425 b/sql/hive/src/test/resources/golden/NaN to Decimal-0-6ca781bc343025635d72321ef0a9d425
new file mode 100644
index 0000000..7951def
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/NaN to Decimal-0-6ca781bc343025635d72321ef0a9d425
@@ -0,0 +1 @@
+NULL
http://git-wip-us.apache.org/repos/asf/spark/blob/a0300ea3/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 684d228..0dd766f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -56,6 +56,9 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
Locale.setDefault(originalLocale)
}
+ createQueryTest("NaN to Decimal",
+ "SELECT CAST(CAST('NaN' AS DOUBLE) AS DECIMAL(1,1)) FROM src LIMIT 1")
+
createQueryTest("constant null testing",
"""SELECT
|IF(FALSE, CAST(NULL AS STRING), CAST(1 AS STRING)) AS COL1,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org