You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2022/11/29 16:37:25 UTC
[spark] branch master updated: [SPARK-41309][SQL] Reuse `INVALID_SCHEMA.NON_STRING_LITERAL` instead of `_LEGACY_ERROR_TEMP_1093`
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a47869af7fa [SPARK-41309][SQL] Reuse `INVALID_SCHEMA.NON_STRING_LITERAL` instead of `_LEGACY_ERROR_TEMP_1093`
a47869af7fa is described below
commit a47869af7fa82b708520da123fa0446214f601c2
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Tue Nov 29 19:36:59 2022 +0300
[SPARK-41309][SQL] Reuse `INVALID_SCHEMA.NON_STRING_LITERAL` instead of `_LEGACY_ERROR_TEMP_1093`
### What changes were proposed in this pull request?
This pr aims reuse `INVALID_SCHEMA.NON_STRING_LITERAL` instead of `_LEGACY_ERROR_TEMP_1093`.
### Why are the changes needed?
Proper names of error classes to improve user experience with Spark SQL.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GitHub Actions
Closes #38830 from LuciferYang/SPARK-41309.
Lead-authored-by: yangjie01 <ya...@baidu.com>
Co-authored-by: YangJie <ya...@baidu.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
core/src/main/resources/error/error-classes.json | 5 -----
.../apache/spark/sql/catalyst/expressions/ExprUtils.scala | 2 +-
.../apache/spark/sql/errors/QueryCompilationErrors.scala | 6 ------
.../test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala | 13 ++++++++-----
.../scala/org/apache/spark/sql/JsonFunctionsSuite.scala | 13 ++++++++-----
5 files changed, 17 insertions(+), 22 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json
index 89728777201..cddb0848765 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -2215,11 +2215,6 @@
"Cannot read table property '<key>' as it's corrupted.<details>."
]
},
- "_LEGACY_ERROR_TEMP_1093" : {
- "message" : [
- "Schema should be specified in DDL format as a string literal or output of the schema_of_json/schema_of_csv functions instead of <expr>."
- ]
- },
"_LEGACY_ERROR_TEMP_1094" : {
"message" : [
"Schema should be struct type but got <dataType>."
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala
index e9084442b22..fbe3d5eb458 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExprUtils.scala
@@ -39,7 +39,7 @@ object ExprUtils extends QueryErrorsBase {
}
} else {
- throw QueryCompilationErrors.schemaNotFoldableError(exp)
+ throw QueryCompilationErrors.unexpectedSchemaTypeError(exp)
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index ce99bf4aa47..e5b1c3c100d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -1009,12 +1009,6 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase {
messageParameters = Map("inputSchema" -> toSQLExpr(exp)))
}
- def schemaNotFoldableError(exp: Expression): Throwable = {
- new AnalysisException(
- errorClass = "_LEGACY_ERROR_TEMP_1093",
- messageParameters = Map("expr" -> exp.sql))
- }
-
def schemaIsNotStructTypeError(dataType: DataType): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1094",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
index 940eaaed6ac..ab4c148da04 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
@@ -357,11 +357,14 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
Seq("""1,"a"""").toDS().select(from_csv($"value", schema, options)),
Row(Row(1, "a")))
- val errMsg = intercept[AnalysisException] {
- Seq(("1", "i int")).toDF("csv", "schema")
- .select(from_csv($"csv", $"schema", options)).collect()
- }.getMessage
- assert(errMsg.contains("Schema should be specified in DDL format as a string literal"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ Seq(("1", "i int")).toDF("csv", "schema")
+ .select(from_csv($"csv", $"schema", options)).collect()
+ },
+ errorClass = "INVALID_SCHEMA.NON_STRING_LITERAL",
+ parameters = Map("inputSchema" -> "\"schema\"")
+ )
checkError(
exception = intercept[AnalysisException] {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index 800a602425b..99d5fc60cad 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -848,11 +848,14 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
Seq("""{"id":1,"city":"Moscow"}""").toDS().select(from_json($"value", schema, options)),
Row(Row(1, "Moscow")))
- val errMsg = intercept[AnalysisException] {
- Seq(("""{"i":1}""", "i int")).toDF("json", "schema")
- .select(from_json($"json", $"schema", options)).collect()
- }.getMessage
- assert(errMsg.contains("Schema should be specified in DDL format as a string literal"))
+ checkError(
+ exception = intercept[AnalysisException] {
+ Seq(("""{"i":1}""", "i int")).toDF("json", "schema")
+ .select(from_json($"json", $"schema", options)).collect()
+ },
+ errorClass = "INVALID_SCHEMA.NON_STRING_LITERAL",
+ parameters = Map("inputSchema" -> "\"schema\"")
+ )
}
test("schema_of_json - infers the schema of foldable JSON string") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org