You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2025/08/20 21:37:19 UTC
(datafusion-comet) branch main updated: fix: [iceberg] Fall back to spark for schemas with empty structs (#2204)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 7d737d73a fix: [iceberg] Fall back to spark for schemas with empty structs (#2204)
7d737d73a is described below
commit 7d737d73a461e0803c0c1bcd2c6856bf2b1c27bf
Author: Andy Grove <ag...@apache.org>
AuthorDate: Wed Aug 20 15:37:16 2025 -0600
fix: [iceberg] Fall back to spark for schemas with empty structs (#2204)
---
spark/src/main/scala/org/apache/comet/DataTypeSupport.scala | 3 ++-
spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala | 2 ++
spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala
index 66eab8c9f..9adf82958 100644
--- a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala
+++ b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala
@@ -54,7 +54,8 @@ trait DataTypeSupport {
TimestampNTZType =>
true
case StructType(fields) =>
- fields.forall(f => isTypeSupported(f.dataType, f.name, fallbackReasons))
+ fields.nonEmpty && fields.forall(f =>
+ isTypeSupported(f.dataType, f.name, fallbackReasons))
case ArrayType(elementType, _) =>
isTypeSupported(elementType, ARRAY_ELEMENT, fallbackReasons)
case MapType(keyType, valueType, _) =>
diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala
index 7ded26f23..aad74064f 100644
--- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala
+++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala
@@ -357,6 +357,8 @@ case class CometScanTypeChecker(scanImpl: String) extends DataTypeSupport {
false
case _: StructType | _: ArrayType | _: MapType if scanImpl == CometConf.SCAN_NATIVE_COMET =>
false
+ case s: StructType if s.fields.isEmpty =>
+ false
case _ =>
super.isTypeSupported(dt, name, fallbackReasons)
}
diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
index 1566e95e2..1e95389c8 100644
--- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -198,7 +198,7 @@ object QueryPlanSerde extends Logging with CometExprShim {
_: DecimalType | _: DateType | _: BooleanType | _: NullType =>
true
case s: StructType if allowComplex =>
- s.fields.map(_.dataType).forall(supportedDataType(_, allowComplex))
+ s.fields.nonEmpty && s.fields.map(_.dataType).forall(supportedDataType(_, allowComplex))
case a: ArrayType if allowComplex =>
supportedDataType(a.elementType, allowComplex)
case m: MapType if allowComplex =>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org