You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/30 04:06:25 UTC
(datafusion-comet) 01/01: fix: route nested DATE to numeric casts in structs and maps through the codegen dispatcher (#6319)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6319-cf8c993b7ac1737d37aa18793997dee95f9eb106
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit e8ff55088cc42684280c6d4716f930a6e906e3c3
Author: Andy Grove <ag...@apache.org>
AuthorDate: Wed Sep 30 04:06:10 2026 +0000
fix: route nested DATE to numeric casts in structs and maps through the codegen dispatcher (#6319)
---
.../org/apache/comet/expressions/CometCast.scala | 24 +++++++--
.../sql-tests/expressions/cast/cast_complex.sql | 63 ++++++++++++++++++++++
.../org/apache/comet/CometNativeCastSuite.scala | 37 +++++++++++++
3 files changed, 119 insertions(+), 5 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala
index d29ef7cd3b..046b159d04 100644
--- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala
+++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala
@@ -238,6 +238,14 @@ object CometCast
canCastFromDouble(toType)
case (from_struct: StructType, to_struct: StructType) =>
from_struct.fields.zip(to_struct.fields).foreach { case (a, b) =>
+ // `convert` replaces a top-level cast that is always null (DATE to a numeric or boolean
+ // type in LEGACY mode) with a null literal, so the native cast never sees one. A struct
+ // field or map entry does reach it, and there DATE to INT reinterprets the day count
+ // (the kernel `unix_date` relies on) while the other targets raise an error. Arrays of
+ // dates have their own rule above.
+ if (isAlwaysCastToNull(a.dataType, b.dataType, evalMode)) {
+ return unsupported(fromType, toType)
+ }
isSupported(a.dataType, b.dataType, timeZoneId, evalMode) match {
case Compatible(_, _) =>
// all good
@@ -248,11 +256,17 @@ object CometCast
Compatible()
case (from_map: MapType, to_map: MapType) =>
// Native cast_map_to_map recursively casts keys and values, so support is
- // determined by whether both inner casts are individually supported.
- isSupported(from_map.keyType, to_map.keyType, timeZoneId, evalMode) match {
- case Compatible(_, _) =>
- isSupported(from_map.valueType, to_map.valueType, timeZoneId, evalMode)
- case other => other
+ // determined by whether both inner casts are individually supported. As with struct
+ // fields, a key or value cast that is always null has no Spark-compatible native kernel.
+ if (isAlwaysCastToNull(from_map.keyType, to_map.keyType, evalMode) ||
+ isAlwaysCastToNull(from_map.valueType, to_map.valueType, evalMode)) {
+ unsupported(fromType, toType)
+ } else {
+ isSupported(from_map.keyType, to_map.keyType, timeZoneId, evalMode) match {
+ case Compatible(_, _) =>
+ isSupported(from_map.valueType, to_map.valueType, timeZoneId, evalMode)
+ case other => other
+ }
}
case (DataTypes.DateType, toType) => canCastFromDate(toType, evalMode)
case _ => unsupported(fromType, toType)
diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_complex.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_complex.sql
index 3898b2ac8d..130c429452 100644
--- a/spark/src/test/resources/sql-tests/expressions/cast/cast_complex.sql
+++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_complex.sql
@@ -149,3 +149,66 @@ query
SELECT cast(arr_struct as string), id
FROM test_cast_complex
ORDER BY id
+
+-- DATE to a numeric or boolean type is always NULL in legacy mode. A top-level cast is folded to a
+-- null literal during planning, but one nested in a struct field or map value reaches the cast
+-- kernel, so these run through the codegen dispatcher.
+statement
+CREATE TABLE test_cast_complex_date(
+ id int,
+ s struct<d:date,label:string>,
+ m map<string,date>,
+ arr_struct array<struct<d:date>>
+) USING parquet
+
+statement
+INSERT INTO test_cast_complex_date VALUES
+ (
+ 1,
+ named_struct('d', date '2024-01-15', 'label', 'first'),
+ map('a', date '2024-01-15', 'b', cast(null as date)),
+ array(named_struct('d', date '2024-01-15'), named_struct('d', cast(null as date)))
+ ),
+ (
+ 2,
+ named_struct('d', cast(null as date), 'label', cast(null as string)),
+ cast(map() as map<string,date>),
+ cast(array() as array<struct<d:date>>)
+ ),
+ (
+ 3,
+ cast(null as struct<d:date,label:string>),
+ cast(null as map<string,date>),
+ cast(null as array<struct<d:date>>)
+ )
+
+query
+SELECT cast(s as struct<d:int,label:string>), id
+FROM test_cast_complex_date
+ORDER BY id
+
+query
+SELECT cast(s as struct<d:bigint,label:string>), cast(s as struct<d:boolean,label:string>), id
+FROM test_cast_complex_date
+ORDER BY id
+
+query
+SELECT cast(s as struct<d:double,label:string>), cast(s as struct<d:decimal(10,2),label:string>), id
+FROM test_cast_complex_date
+ORDER BY id
+
+query
+SELECT cast(m as map<string,int>), cast(m as map<string,bigint>), id
+FROM test_cast_complex_date
+ORDER BY id
+
+query
+SELECT cast(arr_struct as array<struct<d:int>>), cast(arr_struct as array<struct<d:tinyint>>), id
+FROM test_cast_complex_date
+ORDER BY id
+
+-- DATE to TIMESTAMP and STRING inside a struct or map are unaffected
+query
+SELECT cast(s as struct<d:timestamp,label:string>), cast(m as map<string,string>), id
+FROM test_cast_complex_date
+ORDER BY id
diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala
index 431747787b..480fed1429 100644
--- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala
@@ -2235,6 +2235,43 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
}
}
+ test("cast StructType and MapType with DateType to numeric routes through codegen dispatch") {
+ // LEGACY DATE to a numeric or boolean type is always null. `convert` folds the top-level cast
+ // to a null literal, but a struct field or map value reaches the native cast, which returns
+ // the day count for INT and fails for the other targets. Results are covered by
+ // `cast_complex.sql`; this pins the support levels that keep those casts off the native path.
+ def struct(dt: DataType): StructType = StructType(Seq(StructField("d", dt)))
+ val nullResultTypes = Seq(
+ BooleanType,
+ ByteType,
+ ShortType,
+ IntegerType,
+ LongType,
+ FloatType,
+ DoubleType,
+ DecimalType(10, 2))
+ nullResultTypes.foreach { toElementType =>
+ Seq(
+ struct(DateType) -> struct(toElementType),
+ MapType(StringType, DateType) -> MapType(StringType, toElementType),
+ ArrayType(struct(DateType)) -> ArrayType(struct(toElementType))).foreach {
+ case (fromType, toType) =>
+ val level = CometCast.isSupported(fromType, toType, None, CometEvalMode.LEGACY)
+ assert(level.isInstanceOf[Unsupported], s"$fromType to $toType: $level")
+ }
+ }
+ // Other DATE casts nested in a struct or map keep the support level of the element cast.
+ Seq(TimestampType, DataTypes.TimestampNTZType, StringType).foreach { toElementType =>
+ Seq(
+ struct(DateType) -> struct(toElementType),
+ MapType(StringType, DateType) -> MapType(StringType, toElementType)).foreach {
+ case (fromType, toType) =>
+ val level = CometCast.isSupported(fromType, toType, None, CometEvalMode.LEGACY)
+ assert(level.isInstanceOf[Compatible], s"$fromType to $toType: $level")
+ }
+ }
+ }
+
// https://github.com/apache/datafusion-comet/issues/3906
test("cast nested ArrayType to nested ArrayType") {
val types = Seq(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org