You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/20 13:21:56 UTC
(datafusion-comet) 01/01: test: correct signed-zero array fixture comments (#5702) (#6055)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6055-01df17fdb1d0a39ce3cc5ac05b661978e42b7dc1
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 6f755bbea94a00a68f5784641b25ccf875ea4757
Author: Cestercian <ya...@gmail.com>
AuthorDate: Sun Sep 20 13:21:32 2026 +0000
test: correct signed-zero array fixture comments (#5702) (#6055)
NormalizeFloatingNumbers only rewrites grouping keys, join keys, window
partition specs and Distinct. A plain SELECT keeps -0.0 literals intact,
so the literal array cases diverge from Spark for the same reason as the
column-sourced ones. Update the fixture comments and add ignored literal
queries so the skip is honest.
---
.../sql-tests/expressions/array/array_distinct.sql | 22 +++++++++-----
.../sql-tests/expressions/array/array_except.sql | 14 +++++++--
.../expressions/array/array_intersect.sql | 13 ++++++--
.../sql-tests/expressions/array/array_union.sql | 13 ++++++--
.../org/apache/comet/SqlFileTestParserSuite.scala | 35 ++++++++++++++++++++++
5 files changed, 84 insertions(+), 13 deletions(-)
diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql b/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql
index 037c6f198e..a0ca51a11a 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/array_distinct.sql
@@ -136,15 +136,19 @@ SELECT array_distinct(array(CAST('NaN' AS DOUBLE), NULL, CAST('NaN' AS DOUBLE),
query
SELECT array_distinct(array(CAST('Infinity' AS DOUBLE), CAST('-Infinity' AS DOUBLE), CAST('Infinity' AS DOUBLE), 0.0))
--- negative zero (literal). Spark's NormalizeFloatingNumbers rewrites -0.0 to 0.0 at
--- analysis time, so both Spark and Comet collapse it and agree here.
+-- negative zero (literal). NormalizeFloatingNumbers only rewrites grouping keys,
+-- join keys, window partition specs and Distinct; a plain SELECT keeps the -0.0
+-- literal intact. Spark therefore keeps 0.0 and -0.0 distinct while Comet
+-- (DataFusion) collapses them: array_distinct([0.0, -0.0, 1.0]) is [0.0, -0.0, 1.0]
+-- in Spark but [0.0, 1.0] in Comet. Skip until Spark normalizes these zeros
+-- (Spark 4.2+, SPARK-54918).
query ignore(https://issues.apache.org/jira/browse/SPARK-54918)
SELECT array_distinct(array(0.0, double('-0.0'), 1.0))
--- negative zero (column-sourced). NormalizeFloatingNumbers does not touch parquet
--- columns, so Spark keeps -0.0 distinct from 0.0 while Comet (DataFusion) collapses
--- them: array_distinct([0.0, -0.0, 1.0]) is [0.0, -0.0, 1.0] in Spark but [0.0, 1.0]
--- in Comet. Skip until Spark normalizes these zeros (Spark 4.2+, SPARK-54918).
+-- negative zero (column-sourced). Same divergence as the literal case above:
+-- NormalizeFloatingNumbers does not rewrite array-function inputs, so Spark keeps
+-- -0.0 distinct from 0.0 while Comet collapses them. Skip until Spark normalizes
+-- these zeros (Spark 4.2+, SPARK-54918).
statement
CREATE TABLE test_array_distinct_dbl_negzero(arr array<double>) USING parquet
@@ -177,7 +181,11 @@ SELECT array_distinct(arr) FROM test_array_distinct_float
query
SELECT array_distinct(array(CAST('NaN' AS FLOAT), CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT)))
--- negative zero (column-sourced). Same divergence as the double case above, skipped
+-- negative zero (literal). Same divergence as the double literal case above.
+query ignore(https://issues.apache.org/jira/browse/SPARK-54918)
+SELECT array_distinct(array(CAST(0.0 AS FLOAT), float('-0.0'), CAST(1.0 AS FLOAT)))
+
+-- negative zero (column-sourced). Same divergence as the double cases above, skipped
-- until Spark normalizes these zeros (Spark 4.2+, SPARK-54918).
statement
CREATE TABLE test_array_distinct_flt_negzero(arr array<float>) USING parquet
diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql
index afe79948e2..47504b25ad 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/array_except.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/array_except.sql
@@ -72,10 +72,20 @@ INSERT INTO test_except_float VALUES
query
SELECT a, b, array_except(a, b) FROM test_except_float
+-- negative zero (literal). Same signed-zero divergence as the column-sourced
+-- cases below: NormalizeFloatingNumbers does not rewrite array-function inputs,
+-- so a plain SELECT keeps the -0.0 literal intact. Spark keeps -0.0 distinct
+-- from 0.0 while Comet (DataFusion) collapses them. Skip until Spark normalizes
+-- these zeros (Spark 4.2+, SPARK-54918).
+query ignore(https://issues.apache.org/jira/browse/SPARK-54918)
+SELECT array_except(array(0.0, double('-0.0'), 1.0), array(0.0)),
+ array_except(array(0.0, 1.0), array(double('-0.0')))
+
-- negative zero (column-sourced). Spark keeps -0.0 distinct from 0.0 while Comet
-- (DataFusion) collapses them, so array_except([0.0, -0.0, 1.0], [0.0]) is [-0.0, 1.0]
--- in Spark but [1.0] in Comet. NormalizeFloatingNumbers only rewrites literals, not
--- parquet columns. Skip until Spark normalizes these zeros (Spark 4.2+, SPARK-54918).
+-- in Spark but [1.0] in Comet. The divergence is not limited to parquet columns;
+-- the literal case above is skipped for the same reason. Skip until Spark
+-- normalizes these zeros (Spark 4.2+, SPARK-54918).
statement
CREATE TABLE test_except_dbl_negzero(a array<double>, b array<double>) USING parquet
diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql
index 9995d78b5b..e7a5f0a775 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/array_intersect.sql
@@ -157,11 +157,20 @@ INSERT INTO test_intersect_dbl VALUES
query
SELECT a, b, array_intersect(a, b) FROM test_intersect_dbl
+-- negative zero (literal). Same signed-zero membership divergence as the
+-- column-sourced cases below: NormalizeFloatingNumbers does not rewrite
+-- array-function inputs, so a plain SELECT keeps the -0.0 literal intact.
+query ignore(https://issues.apache.org/jira/browse/SPARK-54918)
+SELECT array_intersect(array(double('-0.0')), array(double('0.0'))),
+ array_intersect(array(double('0.0')), array(double('-0.0'))),
+ array_intersect(array(double('-0.0')), array(double('-0.0')))
+
-- Signed-zero membership. Spark keeps -0.0 distinct from 0.0 while Comet (DataFusion)
-- collapses them, so array_intersect([-0.0], [0.0]) is [] in Spark but [0.0] in Comet,
-- and array_intersect([-0.0], [-0.0]) is [-0.0] in Spark but [0.0] in Comet.
--- NormalizeFloatingNumbers only rewrites literals, not parquet columns. Skip until
--- Spark normalizes these zeros (Spark 4.2+, SPARK-54918).
+-- The divergence is not limited to parquet columns; the literal case above is
+-- skipped for the same reason. Skip until Spark normalizes these zeros
+-- (Spark 4.2+, SPARK-54918).
statement
CREATE TABLE test_intersect_flt_negzero(a array<float>, b array<float>) USING parquet
diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql
index 1f3a9dc9dd..33cd636f8a 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/array_union.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/array_union.sql
@@ -144,10 +144,19 @@ INSERT INTO test_union_float VALUES (array(cast(1.0 as float), cast(2.0 as float
query
SELECT a, b, array_union(a, b) FROM test_union_float
+-- negative zero (literal). Same signed-zero divergence as the column-sourced
+-- cases below: NormalizeFloatingNumbers does not rewrite array-function inputs,
+-- so a plain SELECT keeps the -0.0 literal intact. Spark keeps -0.0 distinct
+-- from 0.0 while Comet (DataFusion) collapses them. Skip until Spark normalizes
+-- these zeros (Spark 4.2+, SPARK-54918).
+query ignore(https://issues.apache.org/jira/browse/SPARK-54918)
+SELECT array_union(array(0.0), array(double('-0.0')))
+
-- negative zero (column-sourced). Spark keeps -0.0 distinct from 0.0 while Comet
-- (DataFusion) collapses them, so array_union([0.0], [-0.0]) is [0.0, -0.0] in Spark
--- but [0.0] in Comet. NormalizeFloatingNumbers only rewrites literals, not parquet
--- columns. Skip until Spark normalizes these zeros (Spark 4.2+, SPARK-54918).
+-- but [0.0] in Comet. The divergence is not limited to parquet columns; the
+-- literal case above is skipped for the same reason. Skip until Spark
+-- normalizes these zeros (Spark 4.2+, SPARK-54918).
statement
CREATE TABLE test_union_dbl_negzero(a array<double>, b array<double>) USING parquet
diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala
index 7f3f9ba475..64e0733900 100644
--- a/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala
@@ -19,6 +19,8 @@
package org.apache.comet
+import java.io.File
+
import org.scalatest.funsuite.AnyFunSuite
/**
@@ -156,4 +158,37 @@ class SqlFileTestParserSuite extends AnyFunSuite {
assert(queries.map(_.mode) === Seq(ExpectNative(Seq("abs")), ExpectDispatch(Seq("hypot"))))
assert(queries.map(_.sql) === Seq("SELECT abs(a) FROM t", "SELECT hypot(a, b) FROM t"))
}
+
+ // #5702: NormalizeFloatingNumbers does not rewrite array-function inputs, so a
+ // plain SELECT keeps -0.0 literals intact. The fixtures must skip those literal
+ // cases for the same reason as the column-sourced ones, and must not claim that
+ // Spark and Comet agree on the literal path.
+ test("signed-zero array fixtures skip literals and do not claim Spark agreement") {
+ val names =
+ Seq("array_distinct.sql", "array_except.sql", "array_intersect.sql", "array_union.sql")
+ val stalePhrases = Seq(
+ "both Spark and Comet collapse it and agree here",
+ "only rewrites literals, not parquet columns")
+ names.foreach { name =>
+ val url = getClass.getClassLoader.getResource(s"sql-tests/expressions/array/$name")
+ assert(url != null, s"missing fixture $name")
+ val file = new File(url.toURI)
+ val text = {
+ val src = scala.io.Source.fromFile(file, "UTF-8")
+ try src.mkString
+ finally src.close()
+ }
+ stalePhrases.foreach { phrase =>
+ assert(!text.contains(phrase), s"$name still claims: $phrase")
+ }
+ val ignoredLiterals = SqlFileTestParser.parse(file).records.collect {
+ case SqlQuery(sql, Ignore(_), _)
+ if sql.contains("array(") &&
+ (sql.contains("double('-0.0')") || sql.contains("float('-0.0')")) &&
+ !sql.toLowerCase.contains(" from ") =>
+ sql
+ }
+ assert(ignoredLiterals.nonEmpty, s"$name is missing an ignored signed-zero literal query")
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org