You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/18 04:34:20 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-5235-1adf5e34596814921c7a35e4f339cb47c0aaafa2 created (now f51f5831b0)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-5235-1adf5e34596814921c7a35e4f339cb47c0aaafa2
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at f51f5831b0 fix: normalize signed zero in nested float array comparisons (#5235)
This branch includes the following new commits:
new f51f5831b0 fix: normalize signed zero in nested float array comparisons (#5235)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: fix: normalize signed zero in nested float array comparisons (#5235)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5235-1adf5e34596814921c7a35e4f339cb47c0aaafa2
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit f51f5831b07e11a56e3d083e7c77b1395f53e715
Author: divyank sameer shah <49...@users.noreply.github.com>
AuthorDate: Fri Sep 18 04:33:56 2026 +0000
fix: normalize signed zero in nested float array comparisons (#5235)
* fix: normalize signed zero in nested float array comparisons
Arrow's nested comparator uses total order, where -0.0 and 0.0 are
distinct, but Spark's ordering.equiv treats them as equal, while
still treating NaN as equal to itself. Normalize negative zero in
nested float leaves before building the comparator so
arrays_overlap and array_position match Spark's semantics.
Ref #5191
* fix: normalize floats once per column instead of per row
The comparator was rebuilding the whole float buffer on every
row, since list.value(i) only narrows offsets, not the values
array. Made it quadratic. Hoist normalization above the loop and
skip it entirely for types with no float leaf.
Also switch to the existing normalize_float helper (already used
in hll_plus_plus.rs) instead of a custom -0.0 only version, since
it canonicalizes NaN too. Fixes a case where [[-NaN]] vs [[NaN]]
was returning false.
Addresses review on #5235.
* fix: address second round review feedback
Fix the -0.0 literal in arrays_overlap.sql, it was coerced through
decimal and testing nothing. Add the nested double array and struct
fixtures for arrays_overlap.sql and array_position.sql.
Also add a comment on arrays_overlap_list_generic noting the flat
path must stay untouched, Spark's flat and nested paths disagree on
signed zero, and a comment on the struct position test noting it is
not reachable from SQL yet.
* test: rename total_order tests to spark_equality
These tests assert Spark's equality semantics (-0.0 == 0.0, -NaN ==
NaN), which is the opposite of Arrow's total order. Rename so a
future arrow-rs upgrade doesn't get "fixed" back to total order by
mistake.
* fix: guard nested float normalization at every recursion level
Once an enclosing struct contained a float, normalize_nested_floats
rebuilt every child regardless of whether that child held any float
data. An empty struct child then reached StructArray::new, which cannot
infer a length from zero columns and panics. Iceberg exposes _partition
as struct<> on an unpartitioned table, so arrays_overlap over a struct
element carrying that column failed with a native exception where Spark
returns a result.
Apply the has_float_leaf guard inside normalize_nested_floats rather
than only at the call sites, so float-free subtrees are returned
unchanged. This fixes the panic and keeps the rebuild proportional to
the float data.
---
.../spark-expr/src/array_funcs/array_position.rs | 56 +++-
.../spark-expr/src/array_funcs/arrays_overlap.rs | 119 ++++++++-
native/spark-expr/src/array_funcs/mod.rs | 1 +
.../src/array_funcs/nested_float_normalize.rs | 284 +++++++++++++++++++++
.../sql-tests/expressions/array/array_position.sql | 17 ++
.../sql-tests/expressions/array/arrays_overlap.sql | 34 ++-
6 files changed, 500 insertions(+), 11 deletions(-)
diff --git a/native/spark-expr/src/array_funcs/array_position.rs b/native/spark-expr/src/array_funcs/array_position.rs
index 191091aabf..060501c5f5 100644
--- a/native/spark-expr/src/array_funcs/array_position.rs
+++ b/native/spark-expr/src/array_funcs/array_position.rs
@@ -33,6 +33,8 @@ use num::Float;
use std::cmp::Ordering;
use std::sync::Arc;
+use super::nested_float_normalize::{has_float_leaf, normalize_nested_floats};
+
/// Spark array_position() function that returns the 1-based position of an element in an array.
/// Returns 0 if the element is not found (Spark behavior differs from DataFusion which returns null).
fn spark_array_position(args: &[ColumnarValue]) -> Result<ColumnarValue, DataFusionError> {
@@ -273,7 +275,15 @@ fn position_fallback<O: OffsetSizeTrait>(
let num_rows = list_array.len();
let nulls = combined_nulls(list_array.nulls(), element.nulls());
let mut result = vec![0i64; num_rows];
- let comparator = make_comparator(values.as_ref(), element.as_ref(), SortOptions::default())?;
+ let values_normalized =
+ has_float_leaf(values.data_type()).then(|| normalize_nested_floats(values));
+ let element_normalized =
+ has_float_leaf(element.data_type()).then(|| normalize_nested_floats(element));
+ let comparator = make_comparator(
+ values_normalized.as_ref().unwrap_or(values).as_ref(),
+ element_normalized.as_ref().unwrap_or(element).as_ref(),
+ SortOptions::default(),
+ )?;
for (row_index, w) in offsets.windows(2).enumerate() {
if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) {
@@ -301,8 +311,6 @@ mod tests {
#[test]
fn test_nested_float_and_null_position() -> DataFusionResult<()> {
- // Arrow and the previous ScalarValue fallback distinguish signed zeros, so the second
- // row matches at position 2 rather than position 1.
let values = ListArray::from_iter_primitive::<Float64Type, _, _>([
Some(vec![Some(1.0)]),
Some(vec![Some(f64::NAN)]),
@@ -324,7 +332,47 @@ mod tests {
let result = array_position_inner(&[Arc::new(array), Arc::new(element)])?;
let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
- assert_eq!(result, &Int64Array::from(vec![2, 2, 1]));
+ assert_eq!(result, &Int64Array::from(vec![2, 1, 1]));
+ Ok(())
+ }
+
+ // array_position over array<struct<...>> currently falls back to Spark
+ // (ArraysBase.isTypeSupported rejects StructType, see #1307), so this only
+ // exercises position_fallback directly and isn't reachable from a SQL query.
+ #[test]
+ fn test_struct_float_field_signed_zero_position() -> DataFusionResult<()> {
+ use arrow::array::{Float64Builder, StructBuilder};
+
+ let fields = vec![Arc::new(Field::new("a", DataType::Float64, true))];
+ let mut values_builder =
+ StructBuilder::new(fields.clone(), vec![Box::new(Float64Builder::new())]);
+ for v in [-0.0, 1.0] {
+ values_builder
+ .field_builder::<Float64Builder>(0)
+ .unwrap()
+ .append_value(v);
+ values_builder.append(true);
+ }
+ let values = Arc::new(values_builder.finish());
+ let array = ListArray::new(
+ Arc::new(Field::new("item", values.data_type().clone(), true)),
+ OffsetBuffer::new(vec![0, 2].into()),
+ values,
+ None,
+ );
+
+ let mut element_builder = StructBuilder::new(fields, vec![Box::new(Float64Builder::new())]);
+ element_builder
+ .field_builder::<Float64Builder>(0)
+ .unwrap()
+ .append_value(0.0);
+ element_builder.append(true);
+ let element = element_builder.finish();
+
+ let result = array_position_inner(&[Arc::new(array), Arc::new(element)])?;
+ let result = result.as_any().downcast_ref::<Int64Array>().unwrap();
+ // {-0.0} is the first element and now matches {0.0}, matching Spark.
+ assert_eq!(result, &Int64Array::from(vec![1]));
Ok(())
}
}
diff --git a/native/spark-expr/src/array_funcs/arrays_overlap.rs b/native/spark-expr/src/array_funcs/arrays_overlap.rs
index 4d74718f16..542687c7d1 100644
--- a/native/spark-expr/src/array_funcs/arrays_overlap.rs
+++ b/native/spark-expr/src/array_funcs/arrays_overlap.rs
@@ -51,6 +51,8 @@ use std::hash::Hash;
use std::ops::Range;
use std::sync::Arc;
+use super::nested_float_normalize::{has_float_leaf, normalize_nested_floats};
+
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkArraysOverlap {
signature: Signature,
@@ -395,11 +397,38 @@ where
}
}
+fn normalize_list_element_floats<OffsetSize: OffsetSizeTrait>(
+ list: &GenericListArray<OffsetSize>,
+) -> GenericListArray<OffsetSize> {
+ let field = match list.data_type() {
+ DataType::List(f) | DataType::LargeList(f) => Arc::clone(f),
+ _ => unreachable!("GenericListArray always has List or LargeList data type"),
+ };
+ let normalized_values = normalize_nested_floats(list.values());
+ GenericListArray::new(
+ field,
+ list.offsets().clone(),
+ normalized_values,
+ list.nulls().cloned(),
+ )
+}
+
/// Fallback for nested and otherwise unhandled element types.
+///
+/// note: Spark's flat arrays_overlap (HashSet<Double>) treats -0.0 and 0.0 as different,
+/// only the nested path here treats them as equal. this normalization can't move into the
+/// flat fast path in arrays_overlap_list without breaking that difference.
fn arrays_overlap_list_generic<OffsetSize: OffsetSizeTrait>(
left: &GenericListArray<OffsetSize>,
right: &GenericListArray<OffsetSize>,
) -> Result<ArrayRef> {
+ let left_owned =
+ has_float_leaf(left.values().data_type()).then(|| normalize_list_element_floats(left));
+ let left: &GenericListArray<OffsetSize> = left_owned.as_ref().unwrap_or(left);
+ let right_owned =
+ has_float_leaf(right.values().data_type()).then(|| normalize_list_element_floats(right));
+ let right: &GenericListArray<OffsetSize> = right_owned.as_ref().unwrap_or(right);
+
let len = left.len();
let mut builder = BooleanArray::builder(len);
@@ -517,10 +546,11 @@ fn needs_comparator(dt: &DataType) -> bool {
mod tests {
use super::*;
use arrow::array::{
- Float64Builder, Int32Array, Int32Builder, ListArray, ListBuilder, StructBuilder,
+ Float64Array, Float64Builder, Int32Array, Int32Builder, ListArray, ListBuilder,
+ StructArray, StructBuilder,
};
use arrow::buffer::{NullBuffer, OffsetBuffer};
- use arrow::datatypes::Field;
+ use arrow::datatypes::{Field, Fields};
fn make_list_array(
values: &Int32Array,
@@ -838,9 +868,8 @@ mod tests {
}
#[test]
- fn test_nested_float_total_order() -> Result<()> {
- // Preserve the existing Arrow total-order behavior: NaN matches itself, while signed
- // zeros are distinct.
+ fn test_nested_float_spark_equality() -> Result<()> {
+ // NaN matches itself, and signed zeros are equal, matching Spark.
let left = make_nested_float_list(&[&[f64::NAN]]);
let right = make_nested_float_list(&[&[f64::NAN]]);
let result = arrays_overlap_list::<i32>(&left, &right)?;
@@ -851,7 +880,18 @@ mod tests {
let right = make_nested_float_list(&[&[-0.0]]);
let result = arrays_overlap_list::<i32>(&left, &right)?;
let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
- assert!(!result.value(0));
+ assert!(result.value(0));
+ Ok(())
+ }
+
+ #[test]
+ fn test_nested_float_signed_nan_spark_equality() -> Result<()> {
+ // [[-NaN]] vs [[NaN]] => true
+ let left = make_nested_float_list(&[&[-f64::NAN]]);
+ let right = make_nested_float_list(&[&[f64::NAN]]);
+ let result = arrays_overlap_list::<i32>(&left, &right)?;
+ let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
+ assert!(result.value(0));
Ok(())
}
@@ -1038,6 +1078,73 @@ mod tests {
Ok(())
}
+ /// Build a single-row ListArray of structs: List<Struct<a: Float64>>
+ fn make_struct_float_list(elements: Vec<Option<f64>>) -> ListArray {
+ let fields = vec![Arc::new(Field::new("a", DataType::Float64, true))];
+ let struct_builder =
+ StructBuilder::new(fields.clone(), vec![Box::new(Float64Builder::new())]);
+ let mut list_builder = ListBuilder::new(struct_builder);
+
+ for elem in &elements {
+ let sb = list_builder.values();
+ sb.field_builder::<Float64Builder>(0)
+ .unwrap()
+ .append_option(*elem);
+ sb.append(true);
+ }
+ list_builder.append(true);
+ list_builder.finish()
+ }
+
+ /// Regression test for an empty struct alongside a float field. Iceberg exposes
+ /// `_partition` as `struct<>` on an unpartitioned table, so
+ /// `arrays_overlap(array(named_struct('x', x, 'p', _partition)), ...)` reaches the
+ /// nested path with a zero column struct child. Rebuilding that child used to panic.
+ #[test]
+ fn test_struct_with_empty_struct_field_overlap() -> Result<()> {
+ fn make_list(values: Vec<f64>) -> ListArray {
+ let len = values.len();
+ let x: ArrayRef = Arc::new(Float64Array::from(values));
+ let partition: ArrayRef = Arc::new(StructArray::new_empty_fields(len, None));
+ let fields: Fields = vec![
+ Arc::new(Field::new("x", DataType::Float64, true)),
+ Arc::new(Field::new("p", partition.data_type().clone(), true)),
+ ]
+ .into();
+ let element: ArrayRef =
+ Arc::new(StructArray::new(fields.clone(), vec![x, partition], None));
+ ListArray::new(
+ Arc::new(Field::new("item", DataType::Struct(fields), true)),
+ OffsetBuffer::new((0..=len as i32).collect::<Vec<_>>().into()),
+ element,
+ None,
+ )
+ }
+
+ // One row per element, mirroring `SELECT ... FROM t` over values 1.0 and 2.0.
+ let left = make_list(vec![1.0, 2.0]);
+ let right = make_list(vec![1.0, 2.0]);
+
+ let result = arrays_overlap_list::<i32>(&left, &right)?;
+ let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
+ assert!(result.value(0));
+ assert!(result.value(1));
+ Ok(())
+ }
+
+ #[test]
+ fn test_struct_float_field_signed_zero_overlap() -> Result<()> {
+ // [{-0.0}] vs [{0.0}] => true, matching Spark
+ let left = make_struct_float_list(vec![Some(-0.0)]);
+ let right = make_struct_float_list(vec![Some(0.0)]);
+
+ let result = arrays_overlap_list::<i32>(&left, &right)?;
+ let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
+ assert!(result.is_valid(0));
+ assert!(result.value(0));
+ Ok(())
+ }
+
#[test]
fn test_struct_null_element() -> Result<()> {
// [NULL] vs [{1,2}] => null (null outer element)
diff --git a/native/spark-expr/src/array_funcs/mod.rs b/native/spark-expr/src/array_funcs/mod.rs
index b8877a93dc..2e4df8b317 100644
--- a/native/spark-expr/src/array_funcs/mod.rs
+++ b/native/spark-expr/src/array_funcs/mod.rs
@@ -23,6 +23,7 @@ mod arrays_zip;
mod flatten;
mod get_array_struct_fields;
mod list_extract;
+mod nested_float_normalize;
mod sequence;
mod size;
diff --git a/native/spark-expr/src/array_funcs/nested_float_normalize.rs b/native/spark-expr/src/array_funcs/nested_float_normalize.rs
new file mode 100644
index 0000000000..f70c5ca7f0
--- /dev/null
+++ b/native/spark-expr/src/array_funcs/nested_float_normalize.rs
@@ -0,0 +1,284 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::math_funcs::internal::normalize_float;
+use arrow::array::{
+ Array, ArrayRef, AsArray, FixedSizeListArray, Float32Array, Float64Array, LargeListArray,
+ ListArray, StructArray,
+};
+use arrow::datatypes::{DataType, Float32Type, Float64Type};
+use std::sync::Arc;
+
+pub(super) fn has_float_leaf(dt: &DataType) -> bool {
+ match dt {
+ DataType::Float32 | DataType::Float64 => true,
+ DataType::List(field) | DataType::LargeList(field) | DataType::FixedSizeList(field, _) => {
+ has_float_leaf(field.data_type())
+ }
+ DataType::Struct(fields) => fields.iter().any(|f| has_float_leaf(f.data_type())),
+ _ => false,
+ }
+}
+
+/// Recursively rebuilds nested arrays with `-0.0` normalized to `0.0` and NaN canonicalized
+/// in any Float32/Float64 leaves.
+///
+/// Subtrees without a float leaf are returned as is. The guard is applied at every level, not
+/// just by the caller, so that a float-free sibling of a float field is never rebuilt. That
+/// keeps the rebuild proportional to the float data, and it also leaves empty structs alone:
+/// `StructArray::new` cannot infer a length from zero columns and would panic. An empty struct
+/// reaches this code through Iceberg's `_partition` metadata column on an unpartitioned table.
+pub(super) fn normalize_nested_floats(array: &ArrayRef) -> ArrayRef {
+ if !has_float_leaf(array.data_type()) {
+ return Arc::clone(array);
+ }
+
+ match array.data_type() {
+ DataType::Float32 => {
+ let normalized: Float32Array =
+ array.as_primitive::<Float32Type>().unary(normalize_float);
+ Arc::new(normalized)
+ }
+ DataType::Float64 => {
+ let normalized: Float64Array =
+ array.as_primitive::<Float64Type>().unary(normalize_float);
+ Arc::new(normalized)
+ }
+ DataType::List(field) => {
+ let list = array.as_list::<i32>();
+ let normalized_values = normalize_nested_floats(list.values());
+ Arc::new(ListArray::new(
+ Arc::clone(field),
+ list.offsets().clone(),
+ normalized_values,
+ list.nulls().cloned(),
+ ))
+ }
+ DataType::LargeList(field) => {
+ let list = array.as_list::<i64>();
+ let normalized_values = normalize_nested_floats(list.values());
+ Arc::new(LargeListArray::new(
+ Arc::clone(field),
+ list.offsets().clone(),
+ normalized_values,
+ list.nulls().cloned(),
+ ))
+ }
+ DataType::FixedSizeList(field, size) => {
+ let list = array.as_fixed_size_list();
+ let normalized_values = normalize_nested_floats(list.values());
+ Arc::new(FixedSizeListArray::new(
+ Arc::clone(field),
+ *size,
+ normalized_values,
+ list.nulls().cloned(),
+ ))
+ }
+ DataType::Struct(_) => {
+ let s = array.as_struct();
+ let normalized_columns: Vec<ArrayRef> =
+ s.columns().iter().map(normalize_nested_floats).collect();
+ Arc::new(StructArray::new(
+ s.fields().clone(),
+ normalized_columns,
+ s.nulls().cloned(),
+ ))
+ }
+ _ => Arc::clone(array),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::Float64Builder;
+ use arrow::array::Int32Array;
+ use arrow::array::ListBuilder;
+ use arrow::datatypes::{Field, Fields};
+
+ #[test]
+ fn test_has_float_leaf() {
+ assert!(has_float_leaf(&DataType::Float64));
+ assert!(has_float_leaf(&DataType::List(Arc::new(Field::new(
+ "item",
+ DataType::Float32,
+ true
+ )))));
+ assert!(has_float_leaf(&DataType::Struct(
+ vec![
+ Arc::new(Field::new("a", DataType::Int32, true)),
+ Arc::new(Field::new("b", DataType::Float64, true)),
+ ]
+ .into()
+ )));
+ assert!(!has_float_leaf(&DataType::Int32));
+ assert!(!has_float_leaf(&DataType::List(Arc::new(Field::new(
+ "item",
+ DataType::Int32,
+ true
+ )))));
+ }
+
+ #[test]
+ fn test_normalize_flat_floats() {
+ let arr: ArrayRef = Arc::new(Float64Array::from(vec![
+ Some(-0.0),
+ Some(0.0),
+ Some(f64::NAN),
+ Some(-f64::NAN),
+ None,
+ Some(1.5),
+ ]));
+ let normalized = normalize_nested_floats(&arr);
+ let normalized = normalized.as_primitive::<Float64Type>();
+
+ assert_eq!(normalized.value(0).to_bits(), 0.0f64.to_bits());
+ assert_eq!(normalized.value(1).to_bits(), 0.0f64.to_bits());
+ assert_eq!(normalized.value(2).to_bits(), f64::NAN.to_bits());
+ assert_eq!(normalized.value(3).to_bits(), f64::NAN.to_bits());
+ assert!(normalized.is_null(4));
+ assert_eq!(normalized.value(5), 1.5);
+ }
+
+ #[test]
+ fn test_normalize_nested_list_floats() {
+ let mut builder = ListBuilder::new(Float64Builder::new());
+ builder.values().append_value(-0.0);
+ builder.values().append_value(-f64::NAN);
+ builder.append(true);
+ let arr: ArrayRef = Arc::new(builder.finish());
+
+ let normalized = normalize_nested_floats(&arr);
+ let normalized = normalized.as_list::<i32>();
+ let inner = normalized.value(0);
+ let inner = inner.as_primitive::<Float64Type>();
+
+ assert_eq!(inner.value(0).to_bits(), 0.0f64.to_bits());
+ assert_eq!(inner.value(1).to_bits(), f64::NAN.to_bits());
+ }
+
+ /// An empty struct sibling of a float field must survive normalization. Iceberg exposes
+ /// `_partition` as `struct<>` on an unpartitioned table, and rebuilding it would panic
+ /// because `StructArray::new` cannot infer a length from zero columns.
+ #[test]
+ fn test_normalize_struct_with_empty_struct_sibling() {
+ let x = Float64Array::from(vec![Some(-0.0), Some(1.0)]);
+ let partition = StructArray::new_empty_fields(2, None);
+ let fields = vec![
+ Arc::new(Field::new("x", DataType::Float64, true)),
+ Arc::new(Field::new("p", partition.data_type().clone(), true)),
+ ];
+ let arr: ArrayRef = Arc::new(StructArray::new(
+ fields.into(),
+ vec![Arc::new(x), Arc::new(partition)],
+ None,
+ ));
+
+ let normalized = normalize_nested_floats(&arr);
+ let normalized = normalized.as_struct();
+
+ let col_x = normalized.column(0).as_primitive::<Float64Type>();
+ assert_eq!(col_x.value(0).to_bits(), 0.0f64.to_bits());
+ assert_eq!(col_x.value(1), 1.0);
+
+ let col_p = normalized.column(1).as_struct();
+ assert_eq!(col_p.num_columns(), 0);
+ assert_eq!(col_p.len(), 2);
+ }
+
+ /// A float-free subtree is returned as is rather than rebuilt. The child here is a nested
+ /// struct, which the recursive arms would otherwise rebuild into a fresh array.
+ #[test]
+ fn test_normalize_leaves_float_free_subtree_untouched() {
+ let inner_fields: Fields = vec![Arc::new(Field::new("i", DataType::Int32, true))].into();
+ let inner: ArrayRef = Arc::new(StructArray::new(
+ inner_fields.clone(),
+ vec![Arc::new(Int32Array::from(vec![Some(1), Some(2)]))],
+ None,
+ ));
+ let floats = Float64Array::from(vec![Some(-0.0), Some(1.0)]);
+ let fields: Fields = vec![
+ Arc::new(Field::new("s", DataType::Struct(inner_fields), true)),
+ Arc::new(Field::new("f", DataType::Float64, true)),
+ ]
+ .into();
+ let arr: ArrayRef = Arc::new(StructArray::new(
+ fields,
+ vec![Arc::clone(&inner), Arc::new(floats)],
+ None,
+ ));
+
+ let normalized = normalize_nested_floats(&arr);
+ let normalized = normalized.as_struct();
+
+ assert!(Arc::ptr_eq(normalized.column(0), &inner));
+ assert_eq!(
+ normalized
+ .column(1)
+ .as_primitive::<Float64Type>()
+ .value(0)
+ .to_bits(),
+ 0.0f64.to_bits()
+ );
+ }
+
+ #[test]
+ fn test_normalize_struct_floats() {
+ let a = Float64Array::from(vec![Some(-0.0), Some(1.0)]);
+ let b = Float64Array::from(vec![Some(-f64::NAN), Some(-0.0)]);
+ let fields = vec![
+ Arc::new(Field::new("a", DataType::Float64, true)),
+ Arc::new(Field::new("b", DataType::Float64, true)),
+ ];
+ let arr: ArrayRef = Arc::new(StructArray::new(
+ fields.into(),
+ vec![Arc::new(a), Arc::new(b)],
+ None,
+ ));
+
+ let normalized = normalize_nested_floats(&arr);
+ let normalized = normalized.as_struct();
+ let col_a = normalized.column(0).as_primitive::<Float64Type>();
+ let col_b = normalized.column(1).as_primitive::<Float64Type>();
+
+ assert_eq!(col_a.value(0).to_bits(), 0.0f64.to_bits());
+ assert_eq!(col_a.value(1), 1.0);
+ assert_eq!(col_b.value(0).to_bits(), f64::NAN.to_bits());
+ assert_eq!(col_b.value(1).to_bits(), 0.0f64.to_bits());
+ }
+
+ #[test]
+ fn test_normalize_fixed_size_list_floats() {
+ let values = Float64Array::from(vec![Some(-0.0), Some(-f64::NAN), Some(1.0), Some(-0.0)]);
+ let field = Arc::new(Field::new("item", DataType::Float64, true));
+ let arr: ArrayRef = Arc::new(FixedSizeListArray::new(
+ Arc::clone(&field),
+ 2,
+ Arc::new(values),
+ None,
+ ));
+
+ let normalized = normalize_nested_floats(&arr);
+ let normalized = normalized.as_fixed_size_list();
+ let flat = normalized.values().as_primitive::<Float64Type>();
+
+ assert_eq!(flat.value(0).to_bits(), 0.0f64.to_bits());
+ assert_eq!(flat.value(1).to_bits(), f64::NAN.to_bits());
+ assert_eq!(flat.value(2), 1.0);
+ assert_eq!(flat.value(3).to_bits(), 0.0f64.to_bits());
+ }
+}
diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_position.sql b/spark/src/test/resources/sql-tests/expressions/array/array_position.sql
index 132158ab6d..33c31d84d8 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/array_position.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/array_position.sql
@@ -254,6 +254,23 @@ INSERT INTO test_ap_nested_str VALUES
query
SELECT array_position(arr, val) FROM test_ap_nested_str
+-- nested double array column: -0.0 and 0.0 are equal under Spark's nested ordering
+statement
+CREATE TABLE test_ap_nested_dbl(arr array<array<double>>, val array<double>) USING parquet
+
+statement
+INSERT INTO test_ap_nested_dbl VALUES
+ (array(array(1.0), array(double('-0.0'))), array(double('0.0'))),
+ (array(array(double('-0.0')), array(1.0)), array(double('0.0'))),
+ (array(array(double('0.0')), array(1.0)), array(double('-0.0'))),
+ (array(array(double('NaN'))), array(double('NaN'))),
+ (array(array(1.0)), array(2.0)),
+ (NULL, array(double('0.0'))),
+ (array(array(double('0.0'))), NULL)
+
+query
+SELECT array_position(arr, val) FROM test_ap_nested_dbl
+
-- timestamp arrays
statement
CREATE TABLE test_ap_ts(arr array<timestamp>, val timestamp) USING parquet
diff --git a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql
index b7859a1092..1ddfe14dbc 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/arrays_overlap.sql
@@ -117,7 +117,7 @@ statement
CREATE TABLE test_overlap_dbl(a array<double>, b array<double>) USING parquet
statement
-INSERT INTO test_overlap_dbl VALUES (array(1.0, 2.0), array(2.0, 3.0)), (array(1.0, double('NaN')), array(double('NaN'), 2.0)), (array(double('Infinity'), 1.0), array(double('Infinity'))), (array(double('-Infinity')), array(double('Infinity'))), (array(0.0), array(double('-0.0'))), (array(1.0, NULL), array(2.0, NULL))
+INSERT INTO test_overlap_dbl VALUES (array(1.0, 2.0), array(2.0, 3.0)), (array(1.0, double('NaN')), array(double('NaN'), 2.0)), (array(double('Infinity'), 1.0), array(double('Infinity'))), (array(double('-Infinity')), array(double('Infinity'))), (array(double('0.0')), array(double('-0.0'))), (array(1.0, NULL), array(2.0, NULL))
query
SELECT a, b, arrays_overlap(a, b) FROM test_overlap_dbl
@@ -233,6 +233,38 @@ INSERT INTO test_overlap_nested VALUES (array(array(1, 2), array(3, 4)), array(a
query
SELECT a, b, arrays_overlap(a, b) FROM test_overlap_nested
+-- nested double arrays: Spark's nested path uses ordering.equiv, where -0.0 == 0.0
+statement
+CREATE TABLE test_overlap_nested_dbl(a array<array<double>>, b array<array<double>>) USING parquet
+
+statement
+INSERT INTO test_overlap_nested_dbl VALUES
+ (array(array(double('-0.0'))), array(array(double('0.0')))),
+ (array(array(double('0.0'))), array(array(double('-0.0')))),
+ (array(array(1.0, double('-0.0'))), array(array(1.0, 0.0))),
+ (array(array(double('NaN'))), array(array(double('NaN')))),
+ (array(array(1.0)), array(array(2.0))),
+ (array(array(double('-0.0')), cast(NULL as array<double>)), array(array(double('0.0')))),
+ (array(cast(NULL as array<double>)), array(array(double('0.0'))))
+
+query
+SELECT a, b, arrays_overlap(a, b) FROM test_overlap_nested_dbl
+
+-- struct element with a double field
+statement
+CREATE TABLE test_overlap_struct_dbl(a array<struct<x:double>>, b array<struct<x:double>>) USING parquet
+
+statement
+INSERT INTO test_overlap_struct_dbl VALUES
+ (array(named_struct('x', double('-0.0'))), array(named_struct('x', double('0.0')))),
+ (array(named_struct('x', double('0.0'))), array(named_struct('x', double('-0.0')))),
+ (array(named_struct('x', double('NaN'))), array(named_struct('x', double('NaN')))),
+ (array(named_struct('x', 1.0)), array(named_struct('x', 2.0))),
+ (array(cast(NULL as struct<x:double>)), array(named_struct('x', double('0.0'))))
+
+query
+SELECT a, b, arrays_overlap(a, b) FROM test_overlap_struct_dbl
+
-- struct element arrays
statement
CREATE TABLE test_overlap_struct(a array<struct<x:int, y:int>>, b array<struct<x:int, y:int>>) USING parquet
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org