You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/08/01 00:13:00 UTC
[spark] branch branch-3.3 updated: [SPARK-39857][SQL][TESTS][FOLLOW-UP] Make "translate complex expression" pass with ANSI mode on
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 301d6e39e22 [SPARK-39857][SQL][TESTS][FOLLOW-UP] Make "translate complex expression" pass with ANSI mode on
301d6e39e22 is described below
commit 301d6e39e22297b3771ef502c3204c5fd84f2f9f
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Mon Aug 1 09:12:40 2022 +0900
[SPARK-39857][SQL][TESTS][FOLLOW-UP] Make "translate complex expression" pass with ANSI mode on
### What changes were proposed in this pull request?
This PR fixes `translate complex expression` to pass with ANSI mode on. We do push `Abs` with ANSI mode on (https://github.com/apache/spark/blob/master/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala#L93):
```
[info] - translate complex expression *** FAILED *** (22 milliseconds)
[info] Expected None, but got Some((ABS(cint) - 2) <= 1) (DataSourceV2StrategySuite.scala:325)
[info] org.scalatest.exceptions.TestFailedException:
[info] at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:472)
[info] at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:471)
[info] at org.scalatest.funsuite.AnyFunSuite.newAssertionFailedException(AnyFunSuite.scala:1563)
[info] at org.scalatest.Assertions.assertResult(Assertions.scala:867)
[info] at org.scalatest.Assertions.assertResult$(Assertions.scala:863)
[info] at org.scalatest.funsuite.AnyFunSuite.assertResult(AnyFunSuite.scala:1563)
[info] at org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite.testTranslateFilter(DataSourceV2StrategySuite.scala:325)
[info] at org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite.$anonfun$new$4(DataSourceV2StrategySuite.scala:176)
[info] at org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite.$anonfun$new$4$adapted(DataSourceV2StrategySuite.scala:170)
[info] at scala.collection.immutable.List.foreach(List.scala:431)
[info] at org.apache.spark.sql.execution.datasources.v2.DataSourceV2StrategySuite.$anonfun$new$3(DataSourceV2StrategySuite.scala:170)
[info] at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[info] at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
[info] at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
[info] at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
[info] at org.scalatest.Transformer.apply(Transformer.scala:22)
[info] at org.scalatest.Transformer.apply(Transformer.scala:20)
[info] at org.scalatest.funsuite.AnyFunSuiteLike$$anon$1.apply(AnyFunSuiteLike.scala:190)
[info] at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:204)
[info] at org.scalatest.funsuite.AnyFunSuiteLike.invokeWithFixture$1(AnyFunSuiteLike.scala:188)
[info] at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTest$1(AnyFunSuiteLike.scala:200)
[info] at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
[info] at org.scalatest.funsuite.AnyFunSuiteLike.runTest(AnyFunSuiteLike.scala:200)
[info] at org.scalatest.funsuite.AnyFunSuiteLike.runTest$(AnyFunSuiteLike.scala:182)
[info] at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:65)
[info] at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:234)
[info] at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:227)
[info] at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:65)
```
https://github.com/apache/spark/runs/7595362617?check_suite_focus=true
### Why are the changes needed?
To make the build pass with ANSI mode on.
### Does this PR introduce _any_ user-facing change?
No, test-only.
### How was this patch tested?
Manually ran the unittest with ANSI mode on.
Closes #37349 from HyukjinKwon/SPARK-39857-followup.
Lead-authored-by: Hyukjin Kwon <gu...@apache.org>
Co-authored-by: Hyukjin Kwon <gu...@gmail.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit c211abe970d9e88fd25cd859ea729e630d9491a7)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../datasources/v2/DataSourceV2StrategySuite.scala | 24 +++++++++++-----------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
index 6d355f77615..c6aa13d9009 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
@@ -160,8 +160,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
// ABS(cint) - 2 <= 1
testTranslateFilter(LessThanOrEqual(
// Expressions are not supported
- // Functions such as 'Abs' are not supported
- Subtract(Abs(attrInt), 2), 1), None)
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ Subtract(Abs(attrInt, failOnError = false), 2), 1), None)
// (cin1 > 1 AND cint < 10) OR (cint > 50 AND cint > 100)
testTranslateFilter(Or(
@@ -187,8 +187,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
testTranslateFilter(Or(
And(
GreaterThan(attrInt, 1),
- // Functions such as 'Abs' are not supported
- LessThan(Abs(attrInt), 10)
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ LessThan(Abs(attrInt, failOnError = false), 10)
),
And(
GreaterThan(attrInt, 50),
@@ -198,8 +198,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
testTranslateFilter(Not(And(
Or(
LessThanOrEqual(attrInt, 1),
- // Functions such as 'Abs' are not supported
- GreaterThanOrEqual(Abs(attrInt), 10)
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ GreaterThanOrEqual(Abs(attrInt, failOnError = false), 10)
),
Or(
LessThanOrEqual(attrInt, 50),
@@ -228,8 +228,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
testTranslateFilter(Or(
Or(
EqualTo(attrInt, 1),
- // Functions such as 'Abs' are not supported
- EqualTo(Abs(attrInt), 10)
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ EqualTo(Abs(attrInt, failOnError = false), 10)
),
Or(
GreaterThan(attrInt, 0),
@@ -264,8 +264,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
LessThan(attrInt, 10)
),
And(
- // Functions such as 'Abs' are not supported
- EqualTo(Abs(attrInt), 6),
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ EqualTo(Abs(attrInt, failOnError = false), 6),
IsNotNull(attrInt))), None)
// (cint > 1 OR cint < 10) AND (cint = 6 OR cint IS NOT NULL)
@@ -294,8 +294,8 @@ class DataSourceV2StrategySuite extends PlanTest with SharedSparkSession {
LessThan(attrInt, 10)
),
Or(
- // Functions such as 'Abs' are not supported
- EqualTo(Abs(attrInt), 6),
+ // Functions such as 'Abs' are not pushed down with ANSI mode off
+ EqualTo(Abs(attrInt, failOnError = false), 6),
IsNotNull(attrInt))), None)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org