You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2021/06/24 05:47:55 UTC
[spark] branch master updated: [SPARK-35852][SQL] Use DateAdd
instead of TimeAdd for DateType +/- INTERVAL DAY
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 61bd036 [SPARK-35852][SQL] Use DateAdd instead of TimeAdd for DateType +/- INTERVAL DAY
61bd036 is described below
commit 61bd036cb96fd3a81fc4d131a29d956407956c94
Author: PengLei <18...@189.cn>
AuthorDate: Thu Jun 24 08:47:29 2021 +0300
[SPARK-35852][SQL] Use DateAdd instead of TimeAdd for DateType +/- INTERVAL DAY
### What changes were proposed in this pull request?
We use `DateAdd` to impl `DateType` `+`/`-` `INTERVAL DAY`
### Why are the changes needed?
To improve the impl of `DateType` `+`/`-` `INTERVAL DAY`
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Add ut test
Closes #33033 from Peng-Lei/SPARK-35852.
Authored-by: PengLei <18...@189.cn>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 5 +++++
.../apache/spark/sql/ColumnExpressionSuite.scala | 23 ++++++++++++++++++++++
2 files changed, 28 insertions(+)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 2f142bf..7cb270c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -53,6 +53,7 @@ import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.{PartitionOverwriteMode, StoreAssignmentPolicy}
import org.apache.spark.sql.types._
+import org.apache.spark.sql.types.DayTimeIntervalType.DAY
import org.apache.spark.sql.util.{CaseInsensitiveStringMap, SchemaUtils}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
@@ -349,7 +350,9 @@ class Analyzer(override val catalogManager: CatalogManager)
case p: LogicalPlan => p.transformExpressionsUpWithPruning(
_.containsPattern(BINARY_ARITHMETIC), ruleId) {
case a @ Add(l, r, f) if a.childrenResolved => (l.dataType, r.dataType) match {
+ case (DateType, DayTimeIntervalType(DAY, DAY)) => DateAdd(l, ExtractANSIIntervalDays(r))
case (DateType, _: DayTimeIntervalType) => TimeAdd(Cast(l, TimestampType), r)
+ case (DayTimeIntervalType(DAY, DAY), DateType) => DateAdd(r, ExtractANSIIntervalDays(l))
case (_: DayTimeIntervalType, DateType) => TimeAdd(Cast(r, TimestampType), l)
case (DateType, _: YearMonthIntervalType) => DateAddYMInterval(l, r)
case (_: YearMonthIntervalType, DateType) => DateAddYMInterval(r, l)
@@ -366,6 +369,8 @@ class Analyzer(override val catalogManager: CatalogManager)
case _ => a
}
case s @ Subtract(l, r, f) if s.childrenResolved => (l.dataType, r.dataType) match {
+ case (DateType, DayTimeIntervalType(DAY, DAY)) =>
+ DateAdd(l, UnaryMinus(ExtractANSIIntervalDays(r), f))
case (DateType, _: DayTimeIntervalType) =>
DatetimeSub(l, r, TimeAdd(Cast(l, TimestampType), UnaryMinus(r, f)))
case (DateType, _: YearMonthIntervalType) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index 13fdbf2..b0cd613 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -2881,4 +2881,27 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
}
}
}
+
+ test("SPARK-35852: add/subtract a interval day to/from a date") {
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+ Seq(
+ (LocalDate.of(1, 1, 1), Duration.ofDays(31)),
+ (LocalDate.of(1582, 9, 15), Duration.ofDays(30)),
+ (LocalDate.of(1900, 1, 1), Duration.ofDays(0)),
+ (LocalDate.of(1970, 1, 1), Duration.ofDays(-1)),
+ (LocalDate.of(2021, 3, 14), Duration.ofDays(1)),
+ (LocalDate.of(2020, 12, 31), Duration.ofDays(4 * 30)),
+ (LocalDate.of(2020, 2, 29), Duration.ofDays(365)),
+ (LocalDate.of(10000, 1, 1), Duration.ofDays(-2))
+ ).foreach { case (date, duration) =>
+ val days = duration.toDays
+ val add = date.plusDays(days)
+ val sub = date.minusDays(days)
+ val df = Seq((date, duration)).toDF("start", "diff")
+ .select($"start", $"diff" cast DayTimeIntervalType(DAY) as "diff")
+ .select($"start" + $"diff", $"diff" + $"start", $"start" - $"diff")
+ checkAnswer(df, Row(add, add, sub))
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org