You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2023/06/28 05:22:16 UTC
[spark] branch master updated: [SPARK-43914][SQL] Assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437]
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1c8c47cb55d [SPARK-43914][SQL] Assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437]
1c8c47cb55d is described below
commit 1c8c47cb55da75526fef4dd41ed0734b01e71814
Author: Jiaan Geng <be...@163.com>
AuthorDate: Wed Jun 28 08:22:01 2023 +0300
[SPARK-43914][SQL] Assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437]
### What changes were proposed in this pull request?
The pr aims to assign names to the error class _LEGACY_ERROR_TEMP_[2433-2437].
### Why are the changes needed?
Improve the error framework.
### Does this PR introduce _any_ user-facing change?
'No'.
### How was this patch tested?
Exists test cases updated.
Closes #41476 from beliefer/SPARK-43914.
Authored-by: Jiaan Geng <be...@163.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
core/src/main/resources/error/error-classes.json | 34 ----------
.../sql/catalyst/analysis/CheckAnalysis.scala | 65 +++++++++++--------
.../sql/catalyst/analysis/AnalysisErrorSuite.scala | 74 ++++++++++++++++++++--
.../org/apache/spark/sql/DataFrameSuite.scala | 14 ++++
4 files changed, 120 insertions(+), 67 deletions(-)
diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json
index 342af0ffa6c..e441686432a 100644
--- a/core/src/main/resources/error/error-classes.json
+++ b/core/src/main/resources/error/error-classes.json
@@ -5637,40 +5637,6 @@
"Cannot change nullable column to non-nullable: <fieldName>."
]
},
- "_LEGACY_ERROR_TEMP_2433" : {
- "message" : [
- "Only a single table generating function is allowed in a SELECT clause, found:",
- "<sqlExprs>."
- ]
- },
- "_LEGACY_ERROR_TEMP_2434" : {
- "message" : [
- "Failure when resolving conflicting references in Join:",
- "<plan>",
- "Conflicting attributes: <conflictingAttributes>."
- ]
- },
- "_LEGACY_ERROR_TEMP_2435" : {
- "message" : [
- "Failure when resolving conflicting references in Intersect:",
- "<plan>",
- "Conflicting attributes: <conflictingAttributes>."
- ]
- },
- "_LEGACY_ERROR_TEMP_2436" : {
- "message" : [
- "Failure when resolving conflicting references in Except:",
- "<plan>",
- "Conflicting attributes: <conflictingAttributes>."
- ]
- },
- "_LEGACY_ERROR_TEMP_2437" : {
- "message" : [
- "Failure when resolving conflicting references in AsOfJoin:",
- "<plan>",
- "Conflicting attributes: <conflictingAttributes>."
- ]
- },
"_LEGACY_ERROR_TEMP_2446" : {
"message" : [
"Operation not allowed: <cmd> only works on table with location provided: <tableIdentWithDB>"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 7c0e8f1490d..a0296d27361 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -674,9 +674,8 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
}
case p @ Project(exprs, _) if containsMultipleGenerators(exprs) =>
- p.failAnalysis(
- errorClass = "_LEGACY_ERROR_TEMP_2433",
- messageParameters = Map("sqlExprs" -> exprs.map(_.sql).mkString(",")))
+ val generators = exprs.filter(expr => expr.exists(_.isInstanceOf[Generator]))
+ throw QueryCompilationErrors.moreThanOneGeneratorError(generators, "SELECT")
case p @ Project(projectList, _) =>
projectList.foreach(_.transformDownWithPruning(
@@ -686,36 +685,48 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog with QueryErrorsB
})
case j: Join if !j.duplicateResolved =>
- val conflictingAttributes = j.left.outputSet.intersect(j.right.outputSet)
- j.failAnalysis(
- errorClass = "_LEGACY_ERROR_TEMP_2434",
- messageParameters = Map(
- "plan" -> plan.toString,
- "conflictingAttributes" -> conflictingAttributes.mkString(",")))
+ val conflictingAttributes =
+ j.left.outputSet.intersect(j.right.outputSet).map(toSQLExpr(_)).mkString(", ")
+ throw SparkException.internalError(
+ msg = s"""
+ |Failure when resolving conflicting references in ${j.nodeName}:
+ |${planToString(plan)}
+ |Conflicting attributes: $conflictingAttributes.""".stripMargin,
+ context = j.origin.getQueryContext,
+ summary = j.origin.context.summary)
case i: Intersect if !i.duplicateResolved =>
- val conflictingAttributes = i.left.outputSet.intersect(i.right.outputSet)
- i.failAnalysis(
- errorClass = "_LEGACY_ERROR_TEMP_2435",
- messageParameters = Map(
- "plan" -> plan.toString,
- "conflictingAttributes" -> conflictingAttributes.mkString(",")))
+ val conflictingAttributes =
+ i.left.outputSet.intersect(i.right.outputSet).map(toSQLExpr(_)).mkString(", ")
+ throw SparkException.internalError(
+ msg = s"""
+ |Failure when resolving conflicting references in ${i.nodeName}:
+ |${planToString(plan)}
+ |Conflicting attributes: $conflictingAttributes.""".stripMargin,
+ context = i.origin.getQueryContext,
+ summary = i.origin.context.summary)
case e: Except if !e.duplicateResolved =>
- val conflictingAttributes = e.left.outputSet.intersect(e.right.outputSet)
- e.failAnalysis(
- errorClass = "_LEGACY_ERROR_TEMP_2436",
- messageParameters = Map(
- "plan" -> plan.toString,
- "conflictingAttributes" -> conflictingAttributes.mkString(",")))
+ val conflictingAttributes =
+ e.left.outputSet.intersect(e.right.outputSet).map(toSQLExpr(_)).mkString(", ")
+ throw SparkException.internalError(
+ msg = s"""
+ |Failure when resolving conflicting references in ${e.nodeName}:
+ |${planToString(plan)}
+ |Conflicting attributes: $conflictingAttributes.""".stripMargin,
+ context = e.origin.getQueryContext,
+ summary = e.origin.context.summary)
case j: AsOfJoin if !j.duplicateResolved =>
- val conflictingAttributes = j.left.outputSet.intersect(j.right.outputSet)
- j.failAnalysis(
- errorClass = "_LEGACY_ERROR_TEMP_2437",
- messageParameters = Map(
- "plan" -> plan.toString,
- "conflictingAttributes" -> conflictingAttributes.mkString(",")))
+ val conflictingAttributes =
+ j.left.outputSet.intersect(j.right.outputSet).map(toSQLExpr(_)).mkString(", ")
+ throw SparkException.internalError(
+ msg = s"""
+ |Failure when resolving conflicting references in ${j.nodeName}:
+ |${planToString(plan)}
+ |Conflicting attributes: $conflictingAttributes.""".stripMargin,
+ context = j.origin.getQueryContext,
+ summary = j.origin.context.summary)
// TODO: although map type is not orderable, technically map type should be able to be
// used in equality comparison, remove this type check once we support it.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 5d26fa5ea81..f994c999903 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.{Count, Max}
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.catalyst.plans.{Cross, LeftOuter, RightOuter}
+import org.apache.spark.sql.catalyst.plans.{AsOfJoinDirection, Cross, Inner, LeftOuter, RightOuter}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData}
import org.apache.spark.sql.internal.SQLConf
@@ -781,11 +781,73 @@ class AnalysisErrorSuite extends AnalysisTest {
test("error test for self-join") {
val join = Join(testRelation, testRelation, Cross, None, JoinHint.NONE)
- val error = intercept[AnalysisException] {
- SimpleAnalyzer.checkAnalysis(join)
- }
- assert(error.message.contains("Failure when resolving conflicting references in Join"))
- assert(error.message.contains("Conflicting attributes"))
+ checkError(
+ exception = intercept[SparkException] {
+ SimpleAnalyzer.checkAnalysis(join)
+ },
+ errorClass = "INTERNAL_ERROR",
+ parameters = Map("message" ->
+ """
+ |Failure when resolving conflicting references in Join:
+ |'Join Cross
+ |:- LocalRelation <empty>, [a#x]
+ |+- LocalRelation <empty>, [a#x]
+ |
+ |Conflicting attributes: "a".""".stripMargin))
+ }
+
+ test("error test for self-intersect") {
+ val intersect = Intersect(testRelation, testRelation, true)
+ checkError(
+ exception = intercept[SparkException] {
+ SimpleAnalyzer.checkAnalysis(intersect)
+ },
+ errorClass = "INTERNAL_ERROR",
+ parameters = Map("message" ->
+ """
+ |Failure when resolving conflicting references in Intersect All:
+ |'Intersect All true
+ |:- LocalRelation <empty>, [a#x]
+ |+- LocalRelation <empty>, [a#x]
+ |
+ |Conflicting attributes: "a".""".stripMargin))
+ }
+
+ test("error test for self-except") {
+ val except = Except(testRelation, testRelation, true)
+ checkError(
+ exception = intercept[SparkException] {
+ SimpleAnalyzer.checkAnalysis(except)
+ },
+ errorClass = "INTERNAL_ERROR",
+ parameters = Map("message" ->
+ """
+ |Failure when resolving conflicting references in Except All:
+ |'Except All true
+ |:- LocalRelation <empty>, [a#x]
+ |+- LocalRelation <empty>, [a#x]
+ |
+ |Conflicting attributes: "a".""".stripMargin))
+ }
+
+ test("error test for self-asOfJoin") {
+ val asOfJoin =
+ AsOfJoin(testRelation, testRelation, testRelation.output(0), testRelation.output(0),
+ None, Inner, tolerance = None, allowExactMatches = true,
+ direction = AsOfJoinDirection("backward"))
+ checkError(
+ exception = intercept[SparkException] {
+ SimpleAnalyzer.checkAnalysis(asOfJoin)
+ },
+ errorClass = "INTERNAL_ERROR",
+ parameters = Map("message" ->
+ """
+ |Failure when resolving conflicting references in AsOfJoin:
+ |'AsOfJoin (a#x >= a#x), Inner
+ |:- LocalRelation <empty>, [a#x]
+ |+- LocalRelation <empty>, [a#x]
+ |
+ |Conflicting attributes: "a".""".stripMargin))
}
test("check grouping expression data types") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 99f2ce9706d..6a90b3409d3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -364,6 +364,20 @@ class DataFrameSuite extends QueryTest
Row("a", Seq("a"), 1) :: Nil)
}
+ test("more than one generator in SELECT clause") {
+ val df = Seq((Array("a"), 1)).toDF("a", "b")
+
+ checkError(
+ exception = intercept[AnalysisException] {
+ df.select(explode($"a").as("a"), explode($"a").as("b"))
+ },
+ errorClass = "UNSUPPORTED_GENERATOR.MULTI_GENERATOR",
+ parameters = Map(
+ "clause" -> "SELECT",
+ "num" -> "2",
+ "generators" -> "\"explode(a)\", \"explode(a)\""))
+ }
+
test("sort after generate with join=true") {
val df = Seq((Array("a"), 1)).toDF("a", "b")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org