You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/03/05 09:19:52 UTC
(spark) branch branch-4.0 updated: [SPARK-51388][SQL] Improve SQL fragment propagation in to_timestamp and UNION
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new ebd27c10a85d [SPARK-51388][SQL] Improve SQL fragment propagation in to_timestamp and UNION
ebd27c10a85d is described below
commit ebd27c10a85d476448c2c0851fc4115819a20c77
Author: Vladimir Golubev <vl...@databricks.com>
AuthorDate: Wed Mar 5 17:19:26 2025 +0800
[SPARK-51388][SQL] Improve SQL fragment propagation in to_timestamp and UNION
### What changes were proposed in this pull request?
Improve SQL fragment propagation in to_timestamp and UNION.
### Why are the changes needed?
To improve error messages:
### Does this PR introduce _any_ user-facing change?
Invalid `Cast` error messages become better.
### How was this patch tested?
Golden file tests are regenerated.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50154 from vladimirg-db/vladimir-golubev_data/improve-sql-fragment-in-some-places.
Authored-by: Vladimir Golubev <vl...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 4b4bdcfe1fe9bfd38030b855139a12fc55034083)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../sql/catalyst/analysis/TypeCoercionBase.scala | 13 ++++++++++---
.../catalyst/expressions/datetimeExpressions.scala | 21 +++++++++++++--------
.../results/timestampNTZ/timestamp-ansi.sql.out | 4 +++-
.../native/stringCastAndExpressions.sql.out | 4 +++-
.../sql-tests/results/udf/udf-union.sql.out | 6 +++---
.../test/resources/sql-tests/results/union.sql.out | 6 +++---
6 files changed, 35 insertions(+), 19 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionBase.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionBase.scala
index 294ee93a3c7b..eae7d5a74dbc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionBase.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionBase.scala
@@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{
Unpivot
}
import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
import org.apache.spark.sql.connector.catalog.procedures.BoundProcedure
import org.apache.spark.sql.types.DataType
@@ -142,7 +143,9 @@ abstract class TypeCoercionBase extends TypeCoercionHelper {
case s @ Except(left, right, isAll)
if s.childrenResolved &&
left.output.length == right.output.length && !s.resolved =>
- val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
+ val newChildren: Seq[LogicalPlan] = withOrigin(s.origin) {
+ buildNewChildrenWithWiderTypes(left :: right :: Nil)
+ }
if (newChildren.isEmpty) {
s -> Nil
} else {
@@ -154,7 +157,9 @@ abstract class TypeCoercionBase extends TypeCoercionHelper {
case s @ Intersect(left, right, isAll)
if s.childrenResolved &&
left.output.length == right.output.length && !s.resolved =>
- val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
+ val newChildren: Seq[LogicalPlan] = withOrigin(s.origin) {
+ buildNewChildrenWithWiderTypes(left :: right :: Nil)
+ }
if (newChildren.isEmpty) {
s -> Nil
} else {
@@ -166,7 +171,9 @@ abstract class TypeCoercionBase extends TypeCoercionHelper {
case s: Union
if s.childrenResolved && !s.byName &&
s.children.forall(_.output.length == s.children.head.output.length) && !s.resolved =>
- val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(s.children)
+ val newChildren: Seq[LogicalPlan] = withOrigin(s.origin) {
+ buildNewChildrenWithWiderTypes(s.children)
+ }
if (newChildren.isEmpty) {
s -> Nil
} else {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 67d9aff947cf..4b182607f4da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{ExpressionBuilder, FunctionRegistry}
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
import org.apache.spark.sql.catalyst.trees.TreePattern._
import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, TimestampFormatter}
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
@@ -2104,11 +2105,13 @@ case class ParseToDate(
ansiEnabled: Boolean = SQLConf.get.ansiEnabled)
extends RuntimeReplaceable with ImplicitCastInputTypes with TimeZoneAwareExpression {
- override lazy val replacement: Expression = format.map { f =>
- Cast(GetTimestamp(left, f, TimestampType, "try_to_date", timeZoneId, ansiEnabled), DateType,
- timeZoneId, EvalMode.fromBoolean(ansiEnabled))
- }.getOrElse(Cast(left, DateType, timeZoneId,
- EvalMode.fromBoolean(ansiEnabled))) // backwards compatibility
+ override lazy val replacement: Expression = withOrigin(origin) {
+ format.map { f =>
+ Cast(GetTimestamp(left, f, TimestampType, "try_to_date", timeZoneId, ansiEnabled), DateType,
+ timeZoneId, EvalMode.fromBoolean(ansiEnabled))
+ }.getOrElse(Cast(left, DateType, timeZoneId,
+ EvalMode.fromBoolean(ansiEnabled))) // backwards compatibility
+ }
def this(left: Expression, format: Expression) = {
this(left, Option(format))
@@ -2183,9 +2186,11 @@ case class ParseToTimestamp(
failOnError: Boolean = SQLConf.get.ansiEnabled)
extends RuntimeReplaceable with ImplicitCastInputTypes with TimeZoneAwareExpression {
- override lazy val replacement: Expression = format.map { f =>
- GetTimestamp(left, f, dataType, "try_to_timestamp", timeZoneId, failOnError = failOnError)
- }.getOrElse(Cast(left, dataType, timeZoneId, ansiEnabled = failOnError))
+ override lazy val replacement: Expression = withOrigin(origin) {
+ format.map { f =>
+ GetTimestamp(left, f, dataType, "try_to_timestamp", timeZoneId, failOnError = failOnError)
+ }.getOrElse(Cast(left, dataType, timeZoneId, ansiEnabled = failOnError))
+ }
def this(left: Expression, format: Expression) = {
this(left, Option(format), SQLConf.get.timestampType)
diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out
index 933982f5cff6..eaae069a2a4b 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out
@@ -394,7 +394,9 @@ org.apache.spark.SparkDateTimeException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "fragment" : ""
+ "startIndex" : 8,
+ "stopIndex" : 22,
+ "fragment" : "to_timestamp(1)"
} ]
}
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
index ea17c7f5289f..3d42e866ea17 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
@@ -344,7 +344,9 @@ org.apache.spark.SparkDateTimeException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "fragment" : ""
+ "startIndex" : 8,
+ "stopIndex" : 22,
+ "fragment" : "to_timestamp(a)"
} ]
}
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-union.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-union.sql.out
index 7f2931c3ade0..c61a6f841c88 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-union.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-union.sql.out
@@ -51,9 +51,9 @@ org.apache.spark.SparkNumberFormatException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "startIndex" : 1,
- "stopIndex" : 243,
- "fragment" : "SELECT udf(c1) as c1, udf(c2) as c2\nFROM (SELECT udf(c1) as c1, udf(c2) as c2 FROM t1 WHERE c2 = 'a'\n UNION ALL\n SELECT udf(c1) as c1, udf(c2) as c2 FROM t2\n UNION ALL\n SELECT udf(c1) as c1, udf(c2) as c2 FROM t2)"
+ "startIndex" : 45,
+ "stopIndex" : 172,
+ "fragment" : "SELECT udf(c1) as c1, udf(c2) as c2 FROM t1 WHERE c2 = 'a'\n UNION ALL\n SELECT udf(c1) as c1, udf(c2) as c2 FROM t2"
} ]
}
diff --git a/sql/core/src/test/resources/sql-tests/results/union.sql.out b/sql/core/src/test/resources/sql-tests/results/union.sql.out
index d7db2163f8b8..84e21a535f35 100644
--- a/sql/core/src/test/resources/sql-tests/results/union.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/union.sql.out
@@ -51,9 +51,9 @@ org.apache.spark.SparkNumberFormatException
"queryContext" : [ {
"objectType" : "",
"objectName" : "",
- "startIndex" : 1,
- "stopIndex" : 133,
- "fragment" : "SELECT *\nFROM (SELECT * FROM t1 where c1 = 1\n UNION ALL\n SELECT * FROM t2\n UNION ALL\n SELECT * FROM t2)"
+ "startIndex" : 18,
+ "stopIndex" : 89,
+ "fragment" : "SELECT * FROM t1 where c1 = 1\n UNION ALL\n SELECT * FROM t2"
} ]
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org