You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/11/18 01:28:10 UTC
spark git commit: [SPARK-4425][SQL] Handle NaN or Infinity cast to
Timestamp correctly.
Repository: spark
Updated Branches:
refs/heads/master 3a81a1c9e -> 566c79193
[SPARK-4425][SQL] Handle NaN or Infinity cast to Timestamp correctly.
`Cast` from `NaN` or `Infinity` of `Double` or `Float` to `TimestampType` throws `NumberFormatException`.
Author: Takuya UESHIN <ue...@happy-camper.st>
Closes #3283 from ueshin/issues/SPARK-4425 and squashes the following commits:
14def0c [Takuya UESHIN] Fix Cast to be able to handle NaN or Infinity to TimestampType.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/566c7919
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/566c7919
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/566c7919
Branch: refs/heads/master
Commit: 566c791931645bfaaaf57ee5a15b9ffad534f81e
Parents: 3a81a1c
Author: Takuya UESHIN <ue...@happy-camper.st>
Authored: Mon Nov 17 16:28:07 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Nov 17 16:28:07 2014 -0800
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/expressions/Cast.scala | 14 ++++++++++++--
.../expressions/ExpressionEvaluationSuite.scala | 5 +++++
2 files changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/566c7919/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index b401096..b47865f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -32,6 +32,8 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
override def nullable = (child.dataType, dataType) match {
case (StringType, _: NumericType) => true
case (StringType, TimestampType) => true
+ case (DoubleType, TimestampType) => true
+ case (FloatType, TimestampType) => true
case (StringType, DateType) => true
case (_: NumericType, DateType) => true
case (BooleanType, DateType) => true
@@ -117,10 +119,18 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w
buildCast[Decimal](_, d => decimalToTimestamp(d))
// TimestampWritable.doubleToTimestamp
case DoubleType =>
- buildCast[Double](_, d => decimalToTimestamp(Decimal(d)))
+ buildCast[Double](_, d => try {
+ decimalToTimestamp(Decimal(d))
+ } catch {
+ case _: NumberFormatException => null
+ })
// TimestampWritable.floatToTimestamp
case FloatType =>
- buildCast[Float](_, f => decimalToTimestamp(Decimal(f)))
+ buildCast[Float](_, f => try {
+ decimalToTimestamp(Decimal(f))
+ } catch {
+ case _: NumberFormatException => null
+ })
}
private[this] def decimalToTimestamp(d: Decimal) = {
http://git-wip-us.apache.org/repos/asf/spark/blob/566c7919/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
index 3a6a020..3f5b9f6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
@@ -450,6 +450,11 @@ class ExpressionEvaluationSuite extends FunSuite {
// A test for higher precision than millis
checkEvaluation(Cast(Cast(0.00000001, TimestampType), DoubleType), 0.00000001)
+
+ checkEvaluation(Cast(Literal(Double.NaN), TimestampType), null)
+ checkEvaluation(Cast(Literal(1.0 / 0.0), TimestampType), null)
+ checkEvaluation(Cast(Literal(Float.NaN), TimestampType), null)
+ checkEvaluation(Cast(Literal(1.0f / 0.0f), TimestampType), null)
}
test("null checking") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org