You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/08/01 00:12:51 UTC
[spark] branch master updated: [SPARK-39857][SQL][TESTS][FOLLOW-UP] Make "translate complex expression" pass with ANSI mode on
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c211abe970d [SPARK-39857][SQL][TESTS][FOLLOW-UP] Make "translate complex expression" pass with ANSI mode on
c211abe970d is described below
commit c211abe970d9e88fd25cd859ea729e630d9491a7
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Mon Aug 1 09:12:40 2022 +0900
[SPARK-39857][SQL][TESTS][FOLLOW-UP] Make "translate complex expression" pass with ANSI mode on
### What changes were proposed in this pull request?
This PR fixes `translate complex expression` to pass with ANSI mode on. We do push `Abs` with ANSI mode on (https://github.com/apache/spark/blob/master/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala#L93):
```
[info] - translate complex expression *** FAILED *** (22 milliseconds)
[info] Expected None, but got Some((ABS(cint) - 2) <= 1) (DataSourceV2StrategySuite.scala:325)
[info] org.scalatest.exceptions.TestFailedException:
[info] at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:472)
[info] at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:471)
[info] at org.scalatest.funsuite.AnyFunSuite.newAssertionFailedException(AnyFunSuite.scala:1563)
[info] at org.scalatest.Assertions.assertResult(Assertions.scala:867)
[info] at org.scalatest.Assertions.assertResult$(Assertions.scala:863)
[info] at org.scalatest.funsuite.AnyFunSuite.assertResult(AnyFunSuite.scala:1563)
[info] at org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite.testTranslateFilter(DataSourceV2StrategySuite.scala:325)
[info] at org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite.$anonfun$new$4(DataSourceV2StrategySuite.scala:176)
[info] at org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite.$anonfun$new$4$adapted(DataSourceV2StrategySuite.scala:170)
[info] at scala.collection.immutable.List.foreach(List.scala:431)
[info] at org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite.$anonfun$new$3(DataSourceV2StrategySuite.scala:170)
[info] at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[info] at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
[info] at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
[info] at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
[info] at org.scalatest.Transformer.apply(Transformer.scala:22)
[info] at org.scalatest.Transformer.apply(Transformer.scala:20)
[info] at org.scalatest.funsuite.AnyFunSuiteLike$$anon$1.apply(AnyFunSuiteLike.scala:190)
[info] at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:204)
[info] at org.scalatest.funsuite.AnyFunSuiteLike.invokeWithFixture$1(AnyFunSuiteLike.scala:188)
[info] at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTest$1(AnyFunSuiteLike.scala:200)
[info] at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
[info] at org.scalatest.funsuite.AnyFunSuiteLike.runTest(AnyFunSuiteLike.scala:200)
[info] at org.scalatest.funsuite.AnyFunSuiteLike.runTest$(AnyFunSuiteLike.scala:182)
[info] at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:65)
[info] at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:234)
[info] at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:227)
[info] at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:65)
```
https://github.com/apache/spark/runs/7595362617?check_suite_focus=true
### Why are the changes needed?
To make the build pass with ANSI mode on.
### Does this PR introduce _any_ user-facing change?
No, test-only.
### How was this patch tested?
Manually ran the unittest with ANSI mode on.
Closes #37349 from HyukjinKwon/SPARK-39857-followup.
Lead-authored-by: Hyukjin Kwon <gu...@apache.org>
Co-authored-by: Hyukjin Kwon <gu...@gmail.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../datasources/v2/DataSourceV2StrategySuite.scala | 24 +++++++++++-----------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
index 5fefcadca3e..8d6ffa30a72 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
@@ -172,8 +172,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
// ABS(cint) - 2 <= 1
testTranslateFilter(LessThanOrEqual(
// Expressions are not supported
- // Functions such as 'Abs' are not supported
- Subtract(Abs(attrInt), 2), 1), None)
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ Subtract(Abs(attrInt, failOnError = false), 2), 1), None)
// (cin1 > 1 AND cint < 10) OR (cint > 50 AND cint > 100)
testTranslateFilter(Or(
@@ -199,8 +199,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
testTranslateFilter(Or(
And(
GreaterThan(attrInt, 1),
- // Functions such as 'Abs' are not supported
- LessThan(Abs(attrInt), 10)
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ LessThan(Abs(attrInt, failOnError = false), 10)
),
And(
GreaterThan(attrInt, 50),
@@ -210,8 +210,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
testTranslateFilter(Not(And(
Or(
LessThanOrEqual(attrInt, 1),
- // Functions such as 'Abs' are not supported
- GreaterThanOrEqual(Abs(attrInt), 10)
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ GreaterThanOrEqual(Abs(attrInt, failOnError = false), 10)
),
Or(
LessThanOrEqual(attrInt, 50),
@@ -240,8 +240,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
testTranslateFilter(Or(
Or(
EqualTo(attrInt, 1),
- // Functions such as 'Abs' are not supported
- EqualTo(Abs(attrInt), 10)
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ EqualTo(Abs(attrInt, failOnError = false), 10)
),
Or(
GreaterThan(attrInt, 0),
@@ -276,8 +276,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
LessThan(attrInt, 10)
),
And(
- // Functions such as 'Abs' are not supported
- EqualTo(Abs(attrInt), 6),
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ EqualTo(Abs(attrInt, failOnError = false), 6),
IsNotNull(attrInt))), None)
// (cint > 1 OR cint < 10) AND (cint = 6 OR cint IS NOT NULL)
@@ -306,8 +306,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
LessThan(attrInt, 10)
),
Or(
- // Functions such as 'Abs' are not supported
- EqualTo(Abs(attrInt), 6),
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ EqualTo(Abs(attrInt, failOnError = false), 6),
IsNotNull(attrInt))), None)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org