You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/12 20:30:50 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-5877-d2f73d1da985bacd4907dc0d79cfa0923180538f created (now b69e4f826e)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-5877-d2f73d1da985bacd4907dc0d79cfa0923180538f
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at b69e4f826e test: cover date maps in Parquet temporal fuzz tests (#5877)
This branch includes the following new commits:
new b69e4f826e test: cover date maps in Parquet temporal fuzz tests (#5877)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: test: cover date maps in Parquet temporal fuzz tests (#5877)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5877-d2f73d1da985bacd4907dc0d79cfa0923180538f
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit b69e4f826e0f0c0bbed2d044712febd0a0ca5b63
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Sat Sep 12 19:30:08 2026 +0000
test: cover date maps in Parquet temporal fuzz tests (#5877)
---
.../org/apache/comet/CometFuzzTestSuite.scala | 25 +++++++++++++++++-----
1 file changed, 20 insertions(+), 5 deletions(-)
diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala
index c84db0396b..55fb6010c9 100644
--- a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala
@@ -274,15 +274,14 @@ class CometFuzzTestSuite extends CometFuzzTestBase {
SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outputTimestampType.toString,
SQLConf.SESSION_LOCAL_TIMEZONE.key -> defaultTimezone) {
- // TODO test with MapType
- // https://github.com/apache/datafusion-comet/issues/2945
val schema = StructType(
Seq(
StructField("c0", DataTypes.DateType),
StructField("c1", DataTypes.createArrayType(DataTypes.DateType)),
StructField(
"c2",
- DataTypes.createStructType(Array(StructField("c3", DataTypes.DateType))))))
+ DataTypes.createStructType(Array(StructField("c3", DataTypes.DateType)))),
+ StructField("c4", MapType(DateType, DateType))))
ParquetGenerator.makeParquetFile(
random,
@@ -309,10 +308,26 @@ class CometFuzzTestSuite extends CometFuzzTestBase {
val columns =
df.schema.fields
.filter(f => DataTypeSupport.hasTemporalType(f.dataType))
- .map(_.name)
for (col <- columns) {
- checkSparkAnswer(s"SELECT $col FROM t1 ORDER BY $col")
+ // Maps are not orderable; the answer helper compares unordered results too.
+ val (_, cometPlan) = col.dataType match {
+ case _: MapType =>
+ val query = s"SELECT ${col.name} FROM t1"
+ if (int96TimestampConversion) {
+ checkSparkAnswer(query)
+ } else {
+ checkSparkAnswerAndOperator(query)
+ }
+ case _ =>
+ checkSparkAnswer(s"SELECT ${col.name} FROM t1 ORDER BY ${col.name}")
+ }
+ // INT96 timestamp conversion intentionally disables the Comet extension.
+ if (!int96TimestampConversion) {
+ assert(
+ collectNativeScans(cometPlan).size == 1,
+ s"Expected a Comet scan for ${col.name} in $tz:\n$cometPlan")
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org