You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/02/28 08:20:53 UTC
[spark] branch branch-3.4 updated: [SPARK-42427][SQL][TESTS][FOLLOW-UP] Disable ANSI for one more conv test case in MathFunctionsSuite
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 1cd031a35f3 [SPARK-42427][SQL][TESTS][FOLLOW-UP] Disable ANSI for one more conv test case in MathFunctionsSuite
1cd031a35f3 is described below
commit 1cd031a35f3db15f28c22bd0ca5a830c30dc760f
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Tue Feb 28 17:20:27 2023 +0900
[SPARK-42427][SQL][TESTS][FOLLOW-UP] Disable ANSI for one more conv test case in MathFunctionsSuite
### What changes were proposed in this pull request?
This PR proposes to disable ANSI for one more conv test cases in `MathFunctionsSuite`. They are intentionally testing the behaviours when ANSI is disabled. This is another followup of https://github.com/apache/spark/pull/40117.
### Why are the changes needed?
To make the ANSI tests pass. It currently fails (https://github.com/apache/spark/actions/runs/4277973597/jobs/7447263656):
```
2023-02-28T02:34:53.0298317Z [0m[[0m[0minfo[0m] [0m[0m[32m- SPARK-36229 inconsistently behaviour where returned value is above the 64 char threshold (105 milliseconds)[0m[0m
2023-02-28T02:34:53.0631723Z 02:34:53.062 ERROR org.apache.spark.executor.Executor: Exception in task 0.0 in stage 146.0 (TID 268)
2023-02-28T02:34:53.0632672Z org.apache.spark.SparkArithmeticException: [ARITHMETIC_OVERFLOW] Overflow in function conv(). If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error.
2023-02-28T02:34:53.0633557Z at org.apache.spark.sql.errors.QueryExecutionErrors$.arithmeticOverflowError(QueryExecutionErrors.scala:643)
2023-02-28T02:34:53.0634361Z at org.apache.spark.sql.errors.QueryExecutionErrors$.overflowInConvError(QueryExecutionErrors.scala:315)
2023-02-28T02:34:53.0635124Z at org.apache.spark.sql.catalyst.util.NumberConverter$.encode(NumberConverter.scala:68)
2023-02-28T02:34:53.0711747Z at org.apache.spark.sql.catalyst.util.NumberConverter$.convert(NumberConverter.scala:158)
2023-02-28T02:34:53.0712298Z at org.apache.spark.sql.catalyst.util.NumberConverter.convert(NumberConverter.scala)
2023-02-28T02:34:53.0712925Z at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(generated.java:38)
2023-02-28T02:34:53.0713547Z at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
2023-02-28T02:34:53.0714098Z at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:760)
2023-02-28T02:34:53.0714552Z at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
2023-02-28T02:34:53.0715094Z at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
2023-02-28T02:34:53.0715466Z at org.apache.spark.util.Iterators$.size(Iterators.scala:29)
2023-02-28T02:34:53.0715829Z at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1944)
2023-02-28T02:34:53.0716195Z at org.apache.spark.rdd.RDD.$anonfun$count$1(RDD.scala:1266)
2023-02-28T02:34:53.0716555Z at org.apache.spark.rdd.RDD.$anonfun$count$1$adapted(RDD.scala:1266)
2023-02-28T02:34:53.0716963Z at org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2303)
2023-02-28T02:34:53.0717400Z at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:92)
2023-02-28T02:34:53.0717857Z at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:161)
2023-02-28T02:34:53.0718332Z at org.apache.spark.scheduler.Task.run(Task.scala:139)
2023-02-28T02:34:53.0718743Z at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:554)
2023-02-28T02:34:53.0719152Z at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1520)
2023-02-28T02:34:53.0719548Z at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:557)
2023-02-28T02:34:53.0720001Z at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
2023-02-28T02:34:53.0720481Z at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
2023-02-28T02:34:53.0720848Z at java.lang.Thread.run(Thread.java:750)
2023-02-28T02:34:53.0721492Z 02:34:53.065 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 0.0 in stage 146.0 (TID 268) (localhost executor driver): org.apache.spark.SparkArithmeticException: [ARITHMETIC_OVERFLOW] Overflow in function conv(). If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error.
2023-02-28T02:34:53.0722264Z at org.apache.spark.sql.errors.QueryExecutionErrors$.arithmeticOverflowError(QueryExecutionErrors.scala:643)
2023-02-28T02:34:53.0722821Z at org.apache.spark.sql.errors.QueryExecutionErrors$.overflowInConvError(QueryExecutionErrors.scala:315)
2023-02-28T02:34:53.0723337Z at org.apache.spark.sql.catalyst.util.NumberConverter$.encode(NumberConverter.scala:68)
2023-02-28T02:34:53.0723963Z at org.apache.spark.sql.catalyst.util.NumberConverter$.convert(NumberConverter.scala:158)
2023-02-28T02:34:53.0724474Z at org.apache.spark.sql.catalyst.util.NumberConverter.convert(NumberConverter.scala)
2023-02-28T02:34:53.0725128Z at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(generated.java:38)
2023-02-28T02:34:53.0725826Z at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
2023-02-28T02:34:53.0726376Z at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:760)
2023-02-28T02:34:53.0726827Z at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
2023-02-28T02:34:53.0727189Z at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)
2023-02-28T02:34:53.0727556Z at org.apache.spark.util.Iterators$.size(Iterators.scala:29)
2023-02-28T02:34:53.0727931Z at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1944)
2023-02-28T02:34:53.0728346Z at org.apache.spark.rdd.RDD.$anonfun$count$1(RDD.scala:1266)
2023-02-28T02:34:53.0728701Z at org.apache.spark.rdd.RDD.$anonfun$count$1$adapted(RDD.scala:1266)
2023-02-28T02:34:53.0729096Z at org.apache.spark.SparkContext.$anonfun$runJob$5(SparkContext.scala:2303)
2023-02-28T02:34:53.0729523Z at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:92)
2023-02-28T02:34:53.0729966Z at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:161)
```
### Does this PR introduce _any_ user-facing change?
No, test-only.
### How was this patch tested?
Fixed unittests.
Closes #40209 from HyukjinKwon/SPARK-42427-3.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit 36b64652eb7aa017206d975ca9da58e77bf5e2e4)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../test/scala/org/apache/spark/sql/MathFunctionsSuite.scala | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala
index 01340fa5cda..45b3c379a45 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MathFunctionsSuite.scala
@@ -251,10 +251,12 @@ class MathFunctionsSuite extends QueryTest with SharedSparkSession {
}
test("SPARK-36229 conv should return result equal to -1 in base of toBase") {
- val df = Seq(("aaaaaaa0aaaaaaa0a"), ("aaaaaaa0aaaaaaa0")).toDF("num")
- checkAnswer(df.select(conv($"num", 16, 10)),
- Seq(Row("18446744073709551615"), Row("12297829339523361440")))
- checkAnswer(df.select(conv($"num", 16, -10)), Seq(Row("-1"), Row("-6148914734186190176")))
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) {
+ val df = Seq(("aaaaaaa0aaaaaaa0a"), ("aaaaaaa0aaaaaaa0")).toDF("num")
+ checkAnswer(df.select(conv($"num", 16, 10)),
+ Seq(Row("18446744073709551615"), Row("12297829339523361440")))
+ checkAnswer(df.select(conv($"num", 16, -10)), Seq(Row("-1"), Row("-6148914734186190176")))
+ }
}
test("floor") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org