You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2020/09/16 03:19:49 UTC

[spark] branch branch-3.0 updated: [SPARK-32688][SQL][TEST] Add special values to LiteralGenerator for float and double

This is an automated email from the ASF dual-hosted git repository.

yamamuro pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.0 by this push:
     new cb6a0d0  [SPARK-32688][SQL][TEST] Add special values to LiteralGenerator for float and double
cb6a0d0 is described below

commit cb6a0d08cc020d9a2c19173c9023a9f5e565dd6c
Author: Tanel Kiis <ta...@gmail.com>
AuthorDate: Wed Sep 16 12:13:15 2020 +0900

    [SPARK-32688][SQL][TEST] Add special values to LiteralGenerator for float and double
    
    ### What changes were proposed in this pull request?
    
    The `LiteralGenerator` for float and double datatypes was supposed to yield special values (NaN, +-inf) among others, but the `Gen.chooseNum` method does not yield values that are outside the defined range. The `Gen.chooseNum` for a wide range of floats and doubles does not yield values in the "everyday" range as stated in https://github.com/typelevel/scalacheck/issues/113 .
    
    There is an similar class `RandomDataGenerator` that is used in some other tests. Added `-0.0` and `-0.0f` as special values to there too.
    
    These changes revealed an inconsistency with the equality check between `-0.0` and `0.0`.
    
    ### Why are the changes needed?
    
    The `LiteralGenerator` is mostly used in the `checkConsistencyBetweenInterpretedAndCodegen` method in `MathExpressionsSuite`. This change would have caught the bug fixed in #29495 .
    
    ### Does this PR introduce _any_ user-facing change?
    
    No
    
    ### How was this patch tested?
    
    Locally reverted #29495 and verified that the existing test cases caught the bug.
    
    Closes #29515 from tanelk/SPARK-32688.
    
    Authored-by: Tanel Kiis <ta...@gmail.com>
    Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
    (cherry picked from commit 6051755bfe23a0e4564bf19476ec34cd7fd6008d)
    Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
---
 .../org/apache/spark/sql/RandomDataGenerator.scala    |  4 ++--
 .../sql/catalyst/expressions/LiteralGenerator.scala   | 19 +++++++++++++++----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
index 6a5bdc4..3e2dc3f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
@@ -260,10 +260,10 @@ object RandomDataGenerator {
           new MathContext(precision)).bigDecimal)
       case DoubleType => randomNumeric[Double](
         rand, r => longBitsToDouble(r.nextLong()), Seq(Double.MinValue, Double.MinPositiveValue,
-          Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0))
+          Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0, -0.0))
       case FloatType => randomNumeric[Float](
         rand, r => intBitsToFloat(r.nextInt()), Seq(Float.MinValue, Float.MinPositiveValue,
-          Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f))
+          Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f, -0.0f))
       case ByteType => randomNumeric[Byte](
         rand, _.nextInt().toByte, Seq(Byte.MinValue, Byte.MaxValue, 0.toByte))
       case IntegerType => randomNumeric[Int](
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
index d92eb01..c8e3b0e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
@@ -68,16 +68,27 @@ object LiteralGenerator {
   lazy val longLiteralGen: Gen[Literal] =
     for { l <- Arbitrary.arbLong.arbitrary } yield Literal.create(l, LongType)
 
+  // The floatLiteralGen and doubleLiteralGen will 50% of the time yield arbitrary values
+  // and 50% of the time will yield some special values that are more likely to reveal
+  // corner cases. This behavior is similar to the integral value generators.
   lazy val floatLiteralGen: Gen[Literal] =
     for {
-      f <- Gen.chooseNum(Float.MinValue / 2, Float.MaxValue / 2,
-        Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity)
+      f <- Gen.oneOf(
+        Gen.oneOf(
+          Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity, Float.MinPositiveValue,
+          Float.MaxValue, -Float.MaxValue, 0.0f, -0.0f, 1.0f, -1.0f),
+        Arbitrary.arbFloat.arbitrary
+      )
     } yield Literal.create(f, FloatType)
 
   lazy val doubleLiteralGen: Gen[Literal] =
     for {
-      f <- Gen.chooseNum(Double.MinValue / 2, Double.MaxValue / 2,
-        Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity)
+      f <- Gen.oneOf(
+        Gen.oneOf(
+          Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity, Double.MinPositiveValue,
+          Double.MaxValue, -Double.MaxValue, 0.0, -0.0, 1.0, -1.0),
+        Arbitrary.arbDouble.arbitrary
+      )
     } yield Literal.create(f, DoubleType)
 
   // TODO cache the generated data


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org