You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/06/10 17:28:21 UTC
spark git commit: [SPARK-20211][SQL] Fix the Precision and Scale of
Decimal Values when the Input is BigDecimal between -1.0 and 1.0
Repository: spark
Updated Branches:
refs/heads/master b78e3849b -> 8e96acf71
[SPARK-20211][SQL] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0
### What changes were proposed in this pull request?
The precision and scale of decimal values are wrong when the input is BigDecimal between -1.0 and 1.0.
The BigDecimal's precision is the digit count starts from the leftmost nonzero digit based on the [JAVA's BigDecimal definition](https://docs.oracle.com/javase/7/docs/api/java/math/BigDecimal.html). However, our Decimal decision follows the database decimal standard, which is the total number of digits, including both to the left and the right of the decimal point. Thus, this PR is to fix the issue by doing the conversion.
Before this PR, the following queries failed:
```SQL
select 1 > 0.0001
select floor(0.0001)
select ceil(0.0001)
```
### How was this patch tested?
Added test cases.
Author: Xiao Li <ga...@gmail.com>
Closes #18244 from gatorsmile/bigdecimal.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8e96acf7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8e96acf7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8e96acf7
Branch: refs/heads/master
Commit: 8e96acf71c7bf1686aeca842f626f66c1cc8117f
Parents: b78e384
Author: Xiao Li <ga...@gmail.com>
Authored: Sat Jun 10 10:28:14 2017 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Sat Jun 10 10:28:14 2017 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/types/Decimal.scala | 10 +++-
.../apache/spark/sql/types/DecimalSuite.scala | 10 ++++
.../resources/sql-tests/inputs/operators.sql | 7 +++
.../sql-tests/results/operators.sql.out | 58 +++++++++++++++++---
4 files changed, 75 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8e96acf7/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 80916ee..1f1fb51 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -126,7 +126,15 @@ final class Decimal extends Ordered[Decimal] with Serializable {
def set(decimal: BigDecimal): Decimal = {
this.decimalVal = decimal
this.longVal = 0L
- this._precision = decimal.precision
+ if (decimal.precision <= decimal.scale) {
+ // For Decimal, we expect the precision is equal to or large than the scale, however,
+ // in BigDecimal, the digit count starts from the leftmost nonzero digit of the exact
+ // result. For example, the precision of 0.01 equals to 1 based on the definition, but
+ // the scale is 2. The expected precision should be 3.
+ this._precision = decimal.scale + 1
+ } else {
+ this._precision = decimal.precision
+ }
this._scale = decimal.scale
this
}
http://git-wip-us.apache.org/repos/asf/spark/blob/8e96acf7/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 93c231e..144f3d6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -32,6 +32,16 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester {
test("creating decimals") {
checkDecimal(new Decimal(), "0", 1, 0)
+ checkDecimal(Decimal(BigDecimal("0.09")), "0.09", 3, 2)
+ checkDecimal(Decimal(BigDecimal("0.9")), "0.9", 2, 1)
+ checkDecimal(Decimal(BigDecimal("0.90")), "0.90", 3, 2)
+ checkDecimal(Decimal(BigDecimal("0.0")), "0.0", 2, 1)
+ checkDecimal(Decimal(BigDecimal("0")), "0", 1, 0)
+ checkDecimal(Decimal(BigDecimal("1.0")), "1.0", 2, 1)
+ checkDecimal(Decimal(BigDecimal("-0.09")), "-0.09", 3, 2)
+ checkDecimal(Decimal(BigDecimal("-0.9")), "-0.9", 2, 1)
+ checkDecimal(Decimal(BigDecimal("-0.90")), "-0.90", 3, 2)
+ checkDecimal(Decimal(BigDecimal("-1.0")), "-1.0", 2, 1)
checkDecimal(Decimal(BigDecimal("10.030")), "10.030", 5, 3)
checkDecimal(Decimal(BigDecimal("10.030"), 4, 1), "10.0", 4, 1)
checkDecimal(Decimal(BigDecimal("-9.95"), 4, 1), "-10.0", 4, 1)
http://git-wip-us.apache.org/repos/asf/spark/blob/8e96acf7/sql/core/src/test/resources/sql-tests/inputs/operators.sql
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/inputs/operators.sql b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
index 7e3b86b..75a0256 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/operators.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
@@ -65,8 +65,15 @@ select ceiling(0);
select ceiling(1);
select ceil(1234567890123456);
select ceiling(1234567890123456);
+select ceil(0.01);
+select ceiling(-0.10);
-- floor
select floor(0);
select floor(1);
select floor(1234567890123456);
+select floor(0.01);
+select floor(-0.10);
+
+-- comparison operator
+select 1 > 0.00001
http://git-wip-us.apache.org/repos/asf/spark/blob/8e96acf7/sql/core/src/test/resources/sql-tests/results/operators.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/operators.sql.out b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
index 28cfb74..57e8a61 100644
--- a/sql/core/src/test/resources/sql-tests/results/operators.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 45
+-- Number of queries: 50
-- !query 0
@@ -351,24 +351,64 @@ struct<CEIL(1234567890123456):bigint>
-- !query 42
-select floor(0)
+select ceil(0.01)
-- !query 42 schema
-struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
+struct<CEIL(0.01):decimal(1,0)>
-- !query 42 output
-0
+1
-- !query 43
-select floor(1)
+select ceiling(-0.10)
-- !query 43 schema
-struct<FLOOR(CAST(1 AS DOUBLE)):bigint>
+struct<CEIL(-0.10):decimal(1,0)>
-- !query 43 output
-1
+0
-- !query 44
-select floor(1234567890123456)
+select floor(0)
-- !query 44 schema
-struct<FLOOR(1234567890123456):bigint>
+struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
-- !query 44 output
+0
+
+
+-- !query 45
+select floor(1)
+-- !query 45 schema
+struct<FLOOR(CAST(1 AS DOUBLE)):bigint>
+-- !query 45 output
+1
+
+
+-- !query 46
+select floor(1234567890123456)
+-- !query 46 schema
+struct<FLOOR(1234567890123456):bigint>
+-- !query 46 output
1234567890123456
+
+
+-- !query 47
+select floor(0.01)
+-- !query 47 schema
+struct<FLOOR(0.01):decimal(1,0)>
+-- !query 47 output
+0
+
+
+-- !query 48
+select floor(-0.10)
+-- !query 48 schema
+struct<FLOOR(-0.10):decimal(1,0)>
+-- !query 48 output
+-1
+
+
+-- !query 49
+select 1 > 0.00001
+-- !query 49 schema
+struct<(CAST(1 AS BIGINT) > 0):boolean>
+-- !query 49 output
+true
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org