You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2020/02/05 13:38:22 UTC
[spark] branch branch-3.0 updated: [SPARK-26218][SQL][FOLLOW UP]
Fix the corner case when casting float to Integer
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new fb99ebc [SPARK-26218][SQL][FOLLOW UP] Fix the corner case when casting float to Integer
fb99ebc is described below
commit fb99ebc47d12e82a2814b6b4b3b97148fbaa8627
Author: turbofei <fw...@ebay.com>
AuthorDate: Wed Feb 5 21:24:02 2020 +0800
[SPARK-26218][SQL][FOLLOW UP] Fix the corner case when casting float to Integer
### What changes were proposed in this pull request?
When spark.sql.ansi.enabled is true, for the statement:
```
select cast(cast(2147483648 as Float) as Integer) //result is 2147483647
```
Its result is 2147483647 and does not throw `ArithmeticException`.
The root cause is that, the below code does not work for some corner cases.
https://github.com/apache/spark/blob/94fc0e3235162afc6038019eed6ec546e3d1983e/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala#L129-L141
For example:
In this PR, I fix it by comparing Math.floor(x) with Int.MaxValue directly.
### Why are the changes needed?
Result corrupt.
### Does this PR introduce any user-facing change?
No
### How was this patch tested?
Added Unit test.
Closes #27151 from turboFei/SPARK-26218-follow-up-int-overflow.
Authored-by: turbofei <fw...@ebay.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 6d507b4a31feb965bf31d104f1a6a2c359b166dc)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../main/scala/org/apache/spark/sql/types/numerics.scala | 16 ++++++++--------
.../sql-tests/results/postgreSQL/float4.sql.out | 5 +++--
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 11 +++++++++++
3 files changed, 22 insertions(+), 10 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
index 1ac8536..b522621 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
@@ -121,10 +121,10 @@ object FloatExactNumeric extends FloatIsFractional {
private def overflowException(x: Float, dataType: String) =
throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
- private val intUpperBound = Int.MaxValue.toFloat
- private val intLowerBound = Int.MinValue.toFloat
- private val longUpperBound = Long.MaxValue.toFloat
- private val longLowerBound = Long.MinValue.toFloat
+ private val intUpperBound = Int.MaxValue
+ private val intLowerBound = Int.MinValue
+ private val longUpperBound = Long.MaxValue
+ private val longLowerBound = Long.MinValue
override def toInt(x: Float): Int = {
// When casting floating values to integral types, Spark uses the method `Numeric.toInt`
@@ -155,10 +155,10 @@ object DoubleExactNumeric extends DoubleIsFractional {
private def overflowException(x: Double, dataType: String) =
throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
- private val intUpperBound = Int.MaxValue.toDouble
- private val intLowerBound = Int.MinValue.toDouble
- private val longUpperBound = Long.MaxValue.toDouble
- private val longLowerBound = Long.MinValue.toDouble
+ private val intUpperBound = Int.MaxValue
+ private val intLowerBound = Int.MinValue
+ private val longUpperBound = Long.MaxValue
+ private val longLowerBound = Long.MinValue
override def toInt(x: Double): Int = {
if (Math.floor(x) <= intUpperBound && Math.ceil(x) >= intLowerBound) {
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
index ba91378..fe8375c 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
@@ -322,9 +322,10 @@ struct<CAST(CAST(2147483520 AS FLOAT) AS INT):int>
-- !query
SELECT int(float('2147483647'))
-- !query schema
-struct<CAST(CAST(2147483647 AS FLOAT) AS INT):int>
+struct<>
-- !query output
-2147483647
+java.lang.ArithmeticException
+Casting 2.14748365E9 to int causes overflow
-- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index a6dae9a..11f9724 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3383,6 +3383,17 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
checkAnswer(df, Row(1))
}
}
+
+ test("SPARK-26218: Fix the corner case when casting float to Integer") {
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+ intercept[ArithmeticException](
+ sql("SELECT CAST(CAST(2147483648 as FLOAT) as Integer)").collect()
+ )
+ intercept[ArithmeticException](
+ sql("SELECT CAST(CAST(2147483648 as DOUBLE) as Integer)").collect()
+ )
+ }
+ }
}
case class Foo(bar: Option[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org