You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ru...@apache.org on 2026/08/14 09:54:33 UTC
(gluten) branch main updated: [GLUTEN-11622][VL] Support cast between TIMESTAMP and TIMESTAMP_NTZ (#12654)
This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3ae572bc27 [GLUTEN-11622][VL] Support cast between TIMESTAMP and TIMESTAMP_NTZ (#12654)
3ae572bc27 is described below
commit 3ae572bc2740d028538f549f437557b34f734480
Author: Mariam AlMesfer <Ma...@gmail.com>
AuthorDate: Fri Aug 14 12:54:27 2026 +0300
[GLUTEN-11622][VL] Support cast between TIMESTAMP and TIMESTAMP_NTZ (#12654)
Co-authored-by: Mariam-Almesfer <ma...@ibm.com>
---
.../functions/DateFunctionsValidateSuite.scala | 46 ++++++++++++++++++++++
.../extension/columnar/validator/Validators.scala | 7 +++-
2 files changed, 51 insertions(+), 2 deletions(-)
diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
index 36ce82257a..95619f0e0b 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/functions/DateFunctionsValidateSuite.scala
@@ -615,6 +615,52 @@ class DateFunctionsValidateSuite extends FunctionsValidateSuite {
runQueryAndCompare("select timestampadd(hour, 1, ts) from view") {
checkGlutenPlan[ProjectExecTransformer]
}
+
+ // cast(timestamp_ntz as timestamp)
+ runQueryAndCompare("select cast(ts as timestamp) from view") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+
+ withSQLConf("spark.sql.session.timeZone" -> "Asia/Hong_Kong") {
+ val dstPath = dir.getAbsolutePath + "/dst_gap"
+ spark
+ .createDataset(Seq("1941-12-25 00:00:00"))
+ .toDF("input")
+ .selectExpr("cast(input as timestamp_ntz) as ts")
+ .coalesce(1)
+ .write
+ .mode("overwrite")
+ .parquet(dstPath)
+ spark.read.parquet(dstPath).createOrReplaceTempView("dst_gap_view")
+ runQueryAndCompare("select cast(ts as timestamp) from dst_gap_view") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+ }
+
+ withTempPath {
+ dir =>
+ val path = dir.getAbsolutePath
+ spark
+ .createDataset(inputs)
+ .toDF("input")
+ .selectExpr("cast(input as timestamp) as ts")
+ .coalesce(1)
+ .write
+ .mode("overwrite")
+ .parquet(path)
+ spark.read.parquet(path).createOrReplaceTempView("ts_view")
+
+ // cast(timestamp as timestamp_ntz)
+ runQueryAndCompare("select cast(ts as timestamp_ntz) from ts_view") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+
+ withSQLConf("spark.sql.session.timeZone" -> "UTC") {
+ runQueryAndCompare("select cast(ts as timestamp_ntz) from ts_view") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
}
}
}
diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
index f3825ed89a..b7be523c6d 100644
--- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
+++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala
@@ -25,7 +25,7 @@ import org.apache.gluten.extension.columnar.offload.OffloadSingleNode
import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.expressions.{Hour, Minute, Second, TimestampAdd}
+import org.apache.spark.sql.catalyst.expressions.{Cast, Hour, Minute, Second, TimestampAdd}
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
import org.apache.spark.sql.execution.datasources.WriteFilesExec
@@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleEx
import org.apache.spark.sql.execution.joins._
import org.apache.spark.sql.execution.window.WindowExec
import org.apache.spark.sql.hive.HiveTableScanExecTransformer
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, TimestampType}
object Validators {
implicit class ValidatorBuilderImplicits(builder: Validator.Builder) {
@@ -257,6 +257,7 @@ object Validators {
case mt: MapType => containsNTZ(mt.keyType) || containsNTZ(mt.valueType)
case _ => false
}
+ def isNTZ(dataType: DataType): Boolean = dataType.typeName == "timestamp_ntz"
val hasNTZ = plan.output.exists(a => containsNTZ(a.dataType)) ||
plan.children.exists(_.output.exists(a => containsNTZ(a.dataType)))
if (!hasNTZ) {
@@ -282,6 +283,8 @@ object Validators {
case Minute(child, _) => containsNTZ(child.dataType)
case Second(child, _) => containsNTZ(child.dataType)
case TimestampAdd(_, _, child, _) => containsNTZ(child.dataType)
+ case c: Cast if c.dataType == TimestampType => isNTZ(c.child.dataType)
+ case c: Cast if isNTZ(c.dataType) => c.child.dataType == TimestampType
case _ => false
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org