You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2021/06/24 08:46:58 UTC
[spark] branch branch-3.1 updated: [SPARK-35841][SQL] Casting string to decimal type doesn't work if the…
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new c00f588 [SPARK-35841][SQL] Casting string to decimal type doesn't work if the…
c00f588 is described below
commit c00f58832ab222ebf7ad3b3a61a6f1015679938a
Author: dgd-contributor <dg...@viettel.com.vn>
AuthorDate: Thu Jun 24 16:44:58 2021 +0800
[SPARK-35841][SQL] Casting string to decimal type doesn't work if the…
… sum of the digits is greater than 38
Since Spark 3.1.1, NULL is returned when casting a string with many decimal places to a decimal type. If the sum of the digits before and after the decimal point is less than 39, a value is returned. From 39 digits, however, NULL is returned.
This worked until Spark 3.0.X.
Code to reproduce:
A string with 2 decimal places in front of the decimal point and 37 decimal places after the decimal point returns null
```
val data = Seq(
"28.9259999999999983799625624669715762138",
"28.925999999999998379962562466971576213",
"2.9259999999999983799625624669715762138"
)
val df = data.toDF("num")
df.withColumn("numConverted", col("num").cast("decimal(38, 5)")).show()
```
before this pull request, the result is
+----------------------+---------------+
| num |numConverted|
+----------------------+---------------+
|28.92599999999999...| null|
|28.92599999999999...| 28.92600|
|2.925999999999998...| 2.92600|
+----------------------+---------------+
the correct result should be
+----------------------+---------------+
| num |numConverted|
+----------------------+---------------+
|28.92599999999999...| 28.92600|
|28.92599999999999...| 28.92600|
|2.925999999999998...| 2.92600|
+----------------------+---------------+
The problem occur since https://issues.apache.org/jira/browse/SPARK-32706, it because the fast fail is checking precision length, which should only check the whole number part length of the input value, not the precision length
correctness
no
test added
Closes #33011 from dgd-contributor/SPARK-35841_castStringToDecimalTypeError.
Authored-by: dgd-contributor <dg...@viettel.com.vn>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 5b9c5c126fd391999003f8de9d87c255dfe9424a)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../src/main/scala/org/apache/spark/sql/types/Decimal.scala | 10 +++-------
.../scala/org/apache/spark/sql/types/DecimalSuite.scala | 13 +++++++++++++
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 960e174..72cf03d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -586,12 +586,8 @@ object Decimal {
}
}
- private def calculatePrecision(bigDecimal: JavaBigDecimal): Int = {
- if (bigDecimal.scale < 0) {
+ private def numDigitsInIntegralPart(bigDecimal: JavaBigDecimal): Int = {
bigDecimal.precision - bigDecimal.scale
- } else {
- bigDecimal.precision
- }
}
private def stringToJavaBigDecimal(str: UTF8String): JavaBigDecimal = {
@@ -605,7 +601,7 @@ object Decimal {
val bigDecimal = stringToJavaBigDecimal(str)
// We fast fail because constructing a very large JavaBigDecimal to Decimal is very slow.
// For example: Decimal("6.0790316E+25569151")
- if (calculatePrecision(bigDecimal) > DecimalType.MAX_PRECISION) {
+ if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION) {
null
} else {
Decimal(bigDecimal)
@@ -621,7 +617,7 @@ object Decimal {
val bigDecimal = stringToJavaBigDecimal(str)
// We fast fail because constructing a very large JavaBigDecimal to Decimal is very slow.
// For example: Decimal("6.0790316E+25569151")
- if (calculatePrecision(bigDecimal) > DecimalType.MAX_PRECISION) {
+ if (numDigitsInIntegralPart(bigDecimal) > DecimalType.MAX_PRECISION) {
throw new ArithmeticException(s"out of decimal type range: $str")
} else {
Decimal(bigDecimal)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 7ce451e..57449f5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -286,4 +286,17 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
val e = intercept[NumberFormatException](Decimal.fromStringANSI(UTF8String.fromString("str")))
assert(e.getMessage.contains("invalid input syntax for type numeric"))
}
+
+ test("SPARK-35841: Casting string to decimal type doesn't work " +
+ "if the sum of the digits is greater than 38") {
+ val values = Array(
+ "28.9259999999999983799625624669715762138",
+ "28.925999999999998379962562466971576213",
+ "2.9259999999999983799625624669715762138"
+ )
+ for (string <- values) {
+ assert(Decimal.fromString(UTF8String.fromString(string)) === Decimal(string))
+ assert(Decimal.fromStringANSI(UTF8String.fromString(string)) === Decimal(string))
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org