You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by su...@apache.org on 2026/08/19 16:28:38 UTC
(datafusion-comet) branch main updated: fix: canonicalize NaN in flat arrays_overlap float keys (#5376)
This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new bdd2e13f57 fix: canonicalize NaN in flat arrays_overlap float keys (#5376)
bdd2e13f57 is described below
commit bdd2e13f57c052b2047c2e630bc0a3b1dcd7fe6d
Author: Chao Sun <su...@apache.org>
AuthorDate: Wed Aug 19 09:28:31 2026 -0700
fix: canonicalize NaN in flat arrays_overlap float keys (#5376)
---
.../spark-expr/src/array_funcs/arrays_overlap.rs | 145 ++++++++++++++++++++-
.../sql-tests/expressions/array/arrays_overlap.sql | 51 ++++++++
.../apache/comet/CometArrayExpressionSuite.scala | 27 ++++
3 files changed, 217 insertions(+), 6 deletions(-)
diff --git a/native/spark-expr/src/array_funcs/arrays_overlap.rs b/native/spark-expr/src/array_funcs/arrays_overlap.rs
index bd75a6ddcc..4d74718f16 100644
--- a/native/spark-expr/src/array_funcs/arrays_overlap.rs
+++ b/native/spark-expr/src/array_funcs/arrays_overlap.rs
@@ -272,10 +272,9 @@ fn range_has_null(nulls: Option<&NullBuffer>, range: Range<usize>) -> bool {
nulls.is_some_and(|n| n.null_count() > 0 && n.slice(range.start, range.len()).null_count() > 0)
}
-/// Projects a native value onto a hashable key whose equality matches the Arrow compare kernels:
-/// the value itself for integral types, the bit pattern for floats. Arrow orders floats by total
-/// order rather than IEEE semantics (NaN equals NaN, and 0.0 does not equal -0.0), which is
-/// exactly bit equality.
+/// Projects a native value onto a Spark-compatible hashable key. Floating-point keys canonicalize
+/// every NaN representation while preserving the distinct bit patterns of positive and negative
+/// zero.
trait OverlapKey: Copy {
type Key: Hash + Eq + Copy;
@@ -299,7 +298,11 @@ impl OverlapKey for f32 {
type Key = u32;
fn overlap_key(self) -> u32 {
- self.to_bits()
+ if self.is_nan() {
+ f32::NAN.to_bits()
+ } else {
+ self.to_bits()
+ }
}
}
@@ -307,7 +310,11 @@ impl OverlapKey for f64 {
type Key = u64;
fn overlap_key(self) -> u64 {
- self.to_bits()
+ if self.is_nan() {
+ f64::NAN.to_bits()
+ } else {
+ self.to_bits()
+ }
}
}
@@ -554,6 +561,132 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_flat_float32_nan_payloads_and_signed_zero() -> Result<()> {
+ let positive_nan = f32::from_bits(0x7fc0_0001);
+ let negative_nan = f32::from_bits(0xffc0_0002);
+ let signaling_nan = f32::from_bits(0x7f80_0001);
+
+ let hash_nan_left = (1..=16)
+ .map(|value| Some(value as f32))
+ .chain([Some(positive_nan)])
+ .collect::<Vec<_>>();
+ let hash_nan_right = (17..=32)
+ .map(|value| Some(value as f32))
+ .chain([Some(negative_nan)])
+ .collect::<Vec<_>>();
+ let hash_zero_left = (1..=16)
+ .map(|value| Some(value as f32))
+ .chain([Some(0.0)])
+ .collect::<Vec<_>>();
+ let hash_zero_right = (17..=32)
+ .map(|value| Some(value as f32))
+ .chain([Some(-0.0)])
+ .collect::<Vec<_>>();
+
+ let left = ListArray::from_iter_primitive::<Float32Type, _, _>([
+ Some(vec![Some(positive_nan)]),
+ Some(vec![Some(negative_nan)]),
+ Some(vec![Some(signaling_nan)]),
+ Some(vec![Some(0.0)]),
+ Some(vec![Some(-0.0)]),
+ Some(vec![Some(positive_nan), None]),
+ Some(vec![Some(0.0), None]),
+ Some(hash_nan_left),
+ Some(hash_zero_left),
+ ]);
+ let right = ListArray::from_iter_primitive::<Float32Type, _, _>([
+ Some(vec![Some(f32::NAN)]),
+ Some(vec![Some(positive_nan)]),
+ Some(vec![Some(negative_nan)]),
+ Some(vec![Some(-0.0)]),
+ Some(vec![Some(0.0)]),
+ Some(vec![Some(negative_nan)]),
+ Some(vec![Some(-0.0)]),
+ Some(hash_nan_right),
+ Some(hash_zero_right),
+ ]);
+
+ let result = arrays_overlap_list::<i32>(&left, &right)?;
+ let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
+ let expected = BooleanArray::from(vec![
+ Some(true),
+ Some(true),
+ Some(true),
+ Some(false),
+ Some(false),
+ Some(true),
+ None,
+ Some(true),
+ Some(false),
+ ]);
+ assert_eq!(result, &expected);
+ Ok(())
+ }
+
+ #[test]
+ fn test_flat_float64_nan_payloads_and_signed_zero() -> Result<()> {
+ let positive_nan = f64::from_bits(0x7ff8_0000_0000_0001);
+ let negative_nan = f64::from_bits(0xfff8_0000_0000_0002);
+ let signaling_nan = f64::from_bits(0x7ff0_0000_0000_0001);
+
+ let hash_nan_left = (1..=16)
+ .map(|value| Some(value as f64))
+ .chain([Some(positive_nan)])
+ .collect::<Vec<_>>();
+ let hash_nan_right = (17..=32)
+ .map(|value| Some(value as f64))
+ .chain([Some(negative_nan)])
+ .collect::<Vec<_>>();
+ let hash_zero_left = (1..=16)
+ .map(|value| Some(value as f64))
+ .chain([Some(0.0)])
+ .collect::<Vec<_>>();
+ let hash_zero_right = (17..=32)
+ .map(|value| Some(value as f64))
+ .chain([Some(-0.0)])
+ .collect::<Vec<_>>();
+
+ let left = ListArray::from_iter_primitive::<Float64Type, _, _>([
+ Some(vec![Some(positive_nan)]),
+ Some(vec![Some(negative_nan)]),
+ Some(vec![Some(signaling_nan)]),
+ Some(vec![Some(0.0)]),
+ Some(vec![Some(-0.0)]),
+ Some(vec![Some(positive_nan), None]),
+ Some(vec![Some(0.0), None]),
+ Some(hash_nan_left),
+ Some(hash_zero_left),
+ ]);
+ let right = ListArray::from_iter_primitive::<Float64Type, _, _>([
+ Some(vec![Some(f64::NAN)]),
+ Some(vec![Some(positive_nan)]),
+ Some(vec![Some(negative_nan)]),
+ Some(vec![Some(-0.0)]),
+ Some(vec![Some(0.0)]),
+ Some(vec![Some(negative_nan)]),
+ Some(vec![Some(-0.0)]),
+ Some(hash_nan_right),
+ Some(hash_zero_right),
+ ]);
+
+ let result = arrays_overlap_list::<i32>(&left, &right)?;
+ let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
+ let expected = BooleanArray::from(vec![
+ Some(true),
+ Some(true),
+ Some(true),
+ Some(false),
+ Some(false),
+ Some(true),
+ None,
+ Some(true),
+ Some(false),
+ ]);
+ assert_eq!(result, &expected);
+ Ok(())
+ }
+
#[test]
fn test_null_only_overlap() -> Result<()> {
// [1, NULL] vs [NULL, 2] => null (no definite overlap, but nulls present)
diff --git a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql
index f2a47a33ac..0b3d55366e 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql
@@ -122,6 +122,57 @@ INSERT INTO test_overlap_dbl VALUES (array(1.0, 2.0), array(2.0, 3.0)), (array(1
query
SELECT a, b, arrays_overlap(a, b) FROM test_overlap_dbl
+-- Flat FLOAT/DOUBLE NaNs must compare equal regardless of their sign or payload.
+-- Parquet canonicalizes NaNs, so negate a scanned column to produce a noncanonical
+-- NaN at execution time. String casts preserve the sign of the zero controls.
+statement
+CREATE TABLE test_overlap_floating_point(
+ id int, fl float, fr float, dl double, dr double
+) USING parquet
+
+statement
+INSERT INTO test_overlap_floating_point VALUES
+ (0, CAST('NaN' AS FLOAT), CAST('NaN' AS FLOAT), CAST('NaN' AS DOUBLE), CAST('NaN' AS DOUBLE)),
+ (1, CAST('0.0' AS FLOAT), CAST('-0.0' AS FLOAT), CAST('0.0' AS DOUBLE), CAST('-0.0' AS DOUBLE)),
+ (2, CAST('-0.0' AS FLOAT), CAST('0.0' AS FLOAT), CAST('-0.0' AS DOUBLE), CAST('0.0' AS DOUBLE)),
+ (3, CAST('0.0' AS FLOAT), CAST('0.0' AS FLOAT), CAST('0.0' AS DOUBLE), CAST('0.0' AS DOUBLE)),
+ (4, CAST('-0.0' AS FLOAT), CAST('-0.0' AS FLOAT), CAST('-0.0' AS DOUBLE), CAST('-0.0' AS DOUBLE)),
+ (5, CAST(1 AS FLOAT), CAST(2 AS FLOAT), CAST(1 AS DOUBLE), CAST(2 AS DOUBLE)),
+ (6, CAST('NaN' AS FLOAT), CAST(1 AS FLOAT), CAST('NaN' AS DOUBLE), CAST(1 AS DOUBLE)),
+ (7, CAST(1 AS FLOAT), CAST('NaN' AS FLOAT), CAST(1 AS DOUBLE), CAST('NaN' AS DOUBLE))
+
+-- Short arrays, reversed operands, and direct signed-zero controls. Spark 4.2+
+-- normalizes signed zeros before arrays_overlap (SPARK-54918); comparing with the
+-- running Spark version checks the appropriate behavior without fixed expectations.
+query
+SELECT id,
+ arrays_overlap(array(fl), array(-fr)) AS float_short,
+ arrays_overlap(array(dl), array(-dr)) AS double_short,
+ arrays_overlap(array(-fr), array(fl)) AS float_reversed,
+ arrays_overlap(array(-dr), array(dl)) AS double_reversed,
+ arrays_overlap(array(fl), array(fr)) AS float_direct,
+ arrays_overlap(array(dl), array(dr)) AS double_direct
+FROM test_overlap_floating_point
+
+-- Seventeen elements on each side exceed the native budget of 256 comparisons
+-- and exercise the hash-probe path, including signed-zero and nonmatching rows.
+query
+SELECT id,
+ arrays_overlap(array_repeat(fl, 17), array_repeat(-fr, 17)) AS float_long,
+ arrays_overlap(array_repeat(dl, 17), array_repeat(-dr, 17)) AS double_long,
+ arrays_overlap(array_repeat(-fr, 17), array_repeat(fl, 17)) AS float_long_reversed,
+ arrays_overlap(array_repeat(-dr, 17), array_repeat(dl, 17)) AS double_long_reversed
+FROM test_overlap_floating_point
+
+-- A definite NaN match wins over NULL; a nonmatch with NULL remains unknown.
+query
+SELECT id,
+ arrays_overlap(array(fl, CAST(NULL AS FLOAT)), array(-fr)),
+ arrays_overlap(array(dl, CAST(NULL AS DOUBLE)), array(-dr)),
+ arrays_overlap(array(-fr), array(fl, CAST(NULL AS FLOAT))),
+ arrays_overlap(array(-dr), array(dl, CAST(NULL AS DOUBLE)))
+FROM test_overlap_floating_point
+
-- boolean arrays
query
SELECT arrays_overlap(array(true, false), array(false)) FROM test_overlap_dbl
diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
index 7377f4fe4f..05a6e8e650 100644
--- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
@@ -560,6 +560,33 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp
}
}
+ test("arrays_overlap - runtime NaN representations") {
+ val floatNaN = java.lang.Float.intBitsToFloat(0x7fc01234 | Int.MinValue)
+ val doubleNaN = java.lang.Double.longBitsToDouble(0x7ff8000000001234L | Long.MinValue)
+
+ withParquetTable(
+ Seq((floatNaN, doubleNaN)),
+ "floating_point_overlap",
+ withDictionary = false) {
+ // The behavioral cases live in arrays_overlap.sql. SQL equality cannot distinguish NaN
+ // representations, so verify here that Parquet canonicalizes the inputs and that native
+ // runtime negation produces noncanonical NaNs after the scan.
+ val query = sql("SELECT _1, -_1, _2, -_2 FROM floating_point_overlap")
+ checkSparkAnswerAndOperator(query)
+ val row = query.head()
+ val canonicalFloatNaNBits = java.lang.Float.floatToRawIntBits(Float.NaN)
+ val canonicalDoubleNaNBits = java.lang.Double.doubleToRawLongBits(Double.NaN)
+ assert(java.lang.Float.floatToRawIntBits(row.getFloat(0)) == canonicalFloatNaNBits)
+ assert(
+ java.lang.Float.floatToRawIntBits(row.getFloat(1)) ==
+ (canonicalFloatNaNBits | Int.MinValue))
+ assert(java.lang.Double.doubleToRawLongBits(row.getDouble(2)) == canonicalDoubleNaNBits)
+ assert(
+ java.lang.Double.doubleToRawLongBits(row.getDouble(3)) ==
+ (canonicalDoubleNaNBits | Long.MinValue))
+ }
+ }
+
test("arrays_overlap - null handling behavior verification") {
withSQLConf(
"spark.sql.optimizer.excludedRules" -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org