You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2023/06/30 05:39:04 UTC
[spark] branch master updated: [SPARK-44030][SQL][FOLLOW-UP] Move unapply from AnyTimestampType to AnyTimestampTypeExpression
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 618b52097c0 [SPARK-44030][SQL][FOLLOW-UP] Move unapply from AnyTimestampType to AnyTimestampTypeExpression
618b52097c0 is described below
commit 618b52097c07105d734aaf9b2a22b372920b3f31
Author: Rui Wang <ru...@databricks.com>
AuthorDate: Fri Jun 30 08:38:39 2023 +0300
[SPARK-44030][SQL][FOLLOW-UP] Move unapply from AnyTimestampType to AnyTimestampTypeExpression
### What changes were proposed in this pull request?
Move unapply from AnyTimestampType to AnyTimestampTypeExpression.
### Why are the changes needed?
To align with the effort that we use separate type expression class to host `unapply`.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing Test
Closes #41771 from amaliujia/atomic_datatype_expression.
Authored-by: Rui Wang <ru...@databricks.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 4 ++--
.../spark/sql/catalyst/analysis/AnsiTypeCoercion.scala | 14 ++++++++------
.../apache/spark/sql/catalyst/analysis/TypeCoercion.scala | 12 +++++++-----
.../org/apache/spark/sql/types/AbstractDataType.scala | 3 ---
.../org/apache/spark/sql/types/DataTypeExpression.scala | 5 +++++
5 files changed, 22 insertions(+), 16 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 8a192a4c132..32cec909401 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -428,8 +428,8 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor
UnaryMinus(r, mode == EvalMode.ANSI), ansiEnabled = mode == EvalMode.ANSI))
case (_, CalendarIntervalType | _: DayTimeIntervalType) =>
Cast(DatetimeSub(l, r, TimeAdd(l, UnaryMinus(r, mode == EvalMode.ANSI))), l.dataType)
- case _ if AnyTimestampType.unapply(l) || AnyTimestampType.unapply(r) =>
- SubtractTimestamps(l, r)
+ case _ if AnyTimestampTypeExpression.unapply(l) ||
+ AnyTimestampTypeExpression.unapply(r) => SubtractTimestamps(l, r)
case (_, DateType) => SubtractDates(l, r)
case (DateType, dt) if dt != StringType => DateSub(l, r)
case _ => s
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala
index d3f20f87493..5854f42a061 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala
@@ -284,7 +284,7 @@ object AnsiTypeCoercion extends TypeCoercionBase {
// Skip nodes who's children have not been resolved yet.
case g if !g.childrenResolved => g
- case g: GetDateField if AnyTimestampType.unapply(g.child) =>
+ case g: GetDateField if AnyTimestampTypeExpression.unapply(g.child) =>
g.withNewChildren(Seq(Cast(g.child, DateType)))
}
}
@@ -294,14 +294,16 @@ object AnsiTypeCoercion extends TypeCoercionBase {
// Skip nodes who's children have not been resolved yet.
case e if !e.childrenResolved => e
- case d @ DateAdd(AnyTimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType))
- case d @ DateSub(AnyTimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType))
+ case d @ DateAdd(AnyTimestampTypeExpression(), _) =>
+ d.copy(startDate = Cast(d.startDate, DateType))
+ case d @ DateSub(AnyTimestampTypeExpression(), _) =>
+ d.copy(startDate = Cast(d.startDate, DateType))
- case s @ SubtractTimestamps(DateTypeExpression(), AnyTimestampType(), _, _) =>
+ case s @ SubtractTimestamps(DateTypeExpression(), AnyTimestampTypeExpression(), _, _) =>
s.copy(left = Cast(s.left, s.right.dataType))
- case s @ SubtractTimestamps(AnyTimestampType(), DateTypeExpression(), _, _) =>
+ case s @ SubtractTimestamps(AnyTimestampTypeExpression(), DateTypeExpression(), _, _) =>
s.copy(right = Cast(s.right, s.left.dataType))
- case s @ SubtractTimestamps(AnyTimestampType(), AnyTimestampType(), _, _)
+ case s @ SubtractTimestamps(AnyTimestampTypeExpression(), AnyTimestampTypeExpression(), _, _)
if s.left.dataType != s.right.dataType =>
val newLeft = castIfNotSameType(s.left, TimestampNTZType)
val newRight = castIfNotSameType(s.right, TimestampNTZType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index ae4db0575ad..2a1067be004 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -1174,16 +1174,18 @@ object TypeCoercion extends TypeCoercionBase {
override val transform: PartialFunction[Expression, Expression] = {
// Skip nodes who's children have not been resolved yet.
case e if !e.childrenResolved => e
- case d @ DateAdd(AnyTimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType))
+ case d @ DateAdd(AnyTimestampTypeExpression(), _) =>
+ d.copy(startDate = Cast(d.startDate, DateType))
case d @ DateAdd(StringTypeExpression(), _) => d.copy(startDate = Cast(d.startDate, DateType))
- case d @ DateSub(AnyTimestampType(), _) => d.copy(startDate = Cast(d.startDate, DateType))
+ case d @ DateSub(AnyTimestampTypeExpression(), _) =>
+ d.copy(startDate = Cast(d.startDate, DateType))
case d @ DateSub(StringTypeExpression(), _) => d.copy(startDate = Cast(d.startDate, DateType))
- case s @ SubtractTimestamps(DateTypeExpression(), AnyTimestampType(), _, _) =>
+ case s @ SubtractTimestamps(DateTypeExpression(), AnyTimestampTypeExpression(), _, _) =>
s.copy(left = Cast(s.left, s.right.dataType))
- case s @ SubtractTimestamps(AnyTimestampType(), DateTypeExpression(), _, _) =>
+ case s @ SubtractTimestamps(AnyTimestampTypeExpression(), DateTypeExpression(), _, _) =>
s.copy(right = Cast(s.right, s.left.dataType))
- case s @ SubtractTimestamps(AnyTimestampType(), AnyTimestampType(), _, _)
+ case s @ SubtractTimestamps(AnyTimestampTypeExpression(), AnyTimestampTypeExpression(), _, _)
if s.left.dataType != s.right.dataType =>
val newLeft = castIfNotSameType(s.left, TimestampNTZType)
val newRight = castIfNotSameType(s.right, TimestampNTZType)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
index 01fa27822b0..c1483e719b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/AbstractDataType.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.types
import org.apache.spark.annotation.Stable
-import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.errors.QueryExecutionErrors
/**
@@ -167,8 +166,6 @@ private[sql] object AnyTimestampType extends AbstractDataType with Serializable
other.isInstanceOf[TimestampType] || other.isInstanceOf[TimestampNTZType]
override private[sql] def simpleString = "(timestamp or timestamp without time zone)"
-
- def unapply(e: Expression): Boolean = acceptsType(e.dataType)
}
private[sql] abstract class DatetimeType extends AtomicType
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala
index b2455a41ae6..1b74419a4af 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataTypeExpression.scala
@@ -66,6 +66,11 @@ object IntegralTypeExpression {
}
}
+object AnyTimestampTypeExpression {
+ def unapply(e: Expression): Boolean =
+ e.dataType.isInstanceOf[TimestampType] || e.dataType.isInstanceOf[TimestampNTZType]
+}
+
object DecimalExpression {
def unapply(e: Expression): Option[(Int, Int)] = e.dataType match {
case t: DecimalType => Some((t.precision, t.scale))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org