You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/12 20:30:51 UTC
(datafusion-comet) 01/01: test: cover date maps in Parquet temporal fuzz tests (#5877)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5877-d2f73d1da985bacd4907dc0d79cfa0923180538f
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit b69e4f826e0f0c0bbed2d044712febd0a0ca5b63
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Sat Sep 12 19:30:08 2026 +0000
test: cover date maps in Parquet temporal fuzz tests (#5877)
---
.../org/apache/comet/CometFuzzTestSuite.scala | 25 +++++++++++++++++-----
1 file changed, 20 insertions(+), 5 deletions(-)
diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala
index c84db0396b..55fb6010c9 100644
--- a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala
@@ -274,15 +274,14 @@ class CometFuzzTestSuite extends CometFuzzTestBase {
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outputTimestampType.toString,
SQLConf.SESSION_LOCAL_TIMEZONE.key -> defaultTimezone) {
- // TODO test with MapType
- // https://github.com/apache/datafusion-comet/issues/2945
val schema = StructType(
Seq(
StructField("c0", DataTypes.DateType),
StructField("c1", DataTypes.createArrayType(DataTypes.DateType)),
StructField(
"c2",
- DataTypes.createStructType(Array(StructField("c3", DataTypes.DateType))))))
+ DataTypes.createStructType(Array(StructField("c3", DataTypes.DateType)))),
+ StructField("c4", MapType(DateType, DateType))))
ParquetGenerator.makeParquetFile(
random,
@@ -309,10 +308,26 @@ class CometFuzzTestSuite extends CometFuzzTestBase {
val columns =
df.schema.fields
.filter(f => DataTypeSupport.hasTemporalType(f.dataType))
- .map(_.name)
for (col <- columns) {
- checkSparkAnswer(s"SELECT $col FROM t1 ORDER BY $col")
+ // Maps are not orderable; the answer helper compares unordered results too.
+ val (_, cometPlan) = col.dataType match {
+ case _: MapType =>
+ val query = s"SELECT ${col.name} FROM t1"
+ if (int96TimestampConversion) {
+ checkSparkAnswer(query)
+ } else {
+ checkSparkAnswerAndOperator(query)
+ }
+ case _ =>
+ checkSparkAnswer(s"SELECT ${col.name} FROM t1 ORDER BY ${col.name}")
+ }
+ // INT96 timestamp conversion intentionally disables the Comet extension.
+ if (!int96TimestampConversion) {
+ assert(
+ collectNativeScans(cometPlan).size == 1,
+ s"Expected a Comet scan for ${col.name} in $tz:\n$cometPlan")
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org