You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2022/11/08 10:13:23 UTC
[spark] branch master updated: [SPARK-40984][CORE][SQL] Use `NON_FOLDABLE_INPUT` instead of `FRAME_LESS_OFFSET_WITHOUT_FOLDABLE`
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5c6c5e64ade [SPARK-40984][CORE][SQL] Use `NON_FOLDABLE_INPUT` instead of `FRAME_LESS_OFFSET_WITHOUT_FOLDABLE`
5c6c5e64ade is described below
commit 5c6c5e64ade5310ce84ec9f6eda3a50074324903
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Tue Nov 8 13:13:00 2022 +0300
[SPARK-40984][CORE][SQL] Use `NON_FOLDABLE_INPUT` instead of `FRAME_LESS_OFFSET_WITHOUT_FOLDABLE`
### What changes were proposed in this pull request?
This pr aims replace `FRAME_LESS_OFFSET_WITHOUT_FOLDABLE` with `NON_FOLDABLE_INPUT` to clean up similar error subclass.
### Why are the changes needed?
`FRAME_LESS_OFFSET_WITHOUT_FOLDABLE` and `NON_FOLDABLE_INPUT` look similar, but `NON_FOLDABLE_INPUT` is more general
### Does this PR introduce _any_ user-facing change?
Yes. The PR changes user-facing error messages.
For example, the error message will change from
```
Offset expression "(- nonfoldableliteral())\" must be a literal.
```
to
```
the input offset should be a foldable "INT" expression; however, got "(- nonfoldableliteral())\".
```
### How was this patch tested?
Pass Github Actions
Closes #38536 from LuciferYang/SPARK-40984.
Authored-by: yangjie01 <ya...@baidu.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
core/src/main/resources/error/error-classes.json | 5 -----
.../apache/spark/sql/catalyst/expressions/windowExpressions.scala | 6 ++++--
.../spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala | 8 ++++++--
.../org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala | 7 +++++--
4 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json
index ceb3e4ed5b1..2cbcc165982 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -209,11 +209,6 @@
"Input to <functionName> should all be the same type, but it's <dataType>."
]
},
- "FRAME_LESS_OFFSET_WITHOUT_FOLDABLE" : {
- "message" : [
- "Offset expression <offset> must be a literal."
- ]
- },
"HASH_MAP_TYPE" : {
"message" : [
"Input to the function <functionName> cannot contain elements of the \"MAP\" type. In Spark, same maps may have different hashcode, thus hash expressions are prohibited on \"MAP\" elements. To restore previous behavior set \"spark.sql.legacy.allowHashOnMapType\" to \"true\"."
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index 511a124fd1a..353ab22b5a5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -452,9 +452,11 @@ sealed abstract class FrameLessOffsetWindowFunction
check
} else if (!offset.foldable) {
DataTypeMismatch(
- errorSubClass = "FRAME_LESS_OFFSET_WITHOUT_FOLDABLE",
+ errorSubClass = "NON_FOLDABLE_INPUT",
messageParameters = Map(
- "offset" -> toSQLExpr(offset)
+ "inputName" -> "offset",
+ "inputType" -> toSQLType(offset.dataType),
+ "inputExpr" -> toSQLExpr(offset)
)
)
} else {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index b192f12d569..256cf439b65 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -770,8 +770,12 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
val lag = Lag(Literal(1), NonFoldableLiteral(10), Literal(null), true)
assert(lag.checkInputDataTypes() ==
DataTypeMismatch(
- errorSubClass = "FRAME_LESS_OFFSET_WITHOUT_FOLDABLE",
- messageParameters = Map("offset" -> "\"(- nonfoldableliteral())\"")
+ errorSubClass = "NON_FOLDABLE_INPUT",
+ messageParameters = Map(
+ "inputName" -> "offset",
+ "inputType" -> "\"INT\"",
+ "inputExpr" -> "\"(- nonfoldableliteral())\""
+ )
))
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
index 5a543547708..990c1e1b2de 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
@@ -881,10 +881,13 @@ class DataFrameWindowFunctionsSuite extends QueryTest
lag($"value", 3, null, true).over(window),
lag(concat($"value", $"key"), 1, null, true).over(window)).orderBy($"order").collect()
},
- errorClass = "DATATYPE_MISMATCH.FRAME_LESS_OFFSET_WITHOUT_FOLDABLE",
+ errorClass = "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT",
parameters = Map(
"sqlExpr" -> "\"lag(value, nonfoldableliteral(), NULL)\"",
- "offset" -> "\"(- nonfoldableliteral())\"")
+ "inputName" -> "offset",
+ "inputType" -> "\"INT\"",
+ "inputExpr" -> "\"(- nonfoldableliteral())\""
+ )
)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org