You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/30 13:33:54 UTC
(datafusion) branch gh-readonly-queue/main/pr-25774-884465c2014718cdcacaa908bec125fa68579413 created (now f56ae979e2)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-25774-884465c2014718cdcacaa908bec125fa68579413
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at f56ae979e2 fix: xxhash64 of a NULL struct ignores the struct fields (#25774)
This branch includes the following new commits:
new f56ae979e2 fix: xxhash64 of a NULL struct ignores the struct fields (#25774)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix: xxhash64 of a NULL struct ignores the struct fields (#25774)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25774-884465c2014718cdcacaa908bec125fa68579413
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit f56ae979e29f620fa6effae1836cb4b1492d434e
Author: mrhard9090 <74...@users.noreply.github.com>
AuthorDate: Wed Sep 30 13:33:30 2026 +0000
fix: xxhash64 of a NULL struct ignores the struct fields (#25774)
## Which issue does this PR close?
- Closes #25505.
## Rationale for this change
Spark's `xxhash64` treats a NULL value by leaving the running hash
unchanged, so `xxhash64` of a NULL struct should be the seed, `42`. When
a struct is NULL but its child arrays still hold values, for example
after `nullif(s, s)`, DataFusion hashed those child values anyway:
```sql
SELECT s IS NULL, xxhash64(s), xxhash64(arrow_cast(NULL, 'Struct("a": Int64, "b": Utf8)'))
FROM (SELECT nullif(s, s) AS s FROM (SELECT named_struct('a', 1, 'b', 'x') AS s));
-- true 6158280145716684778 42
```
The same NULL gets two different hashes depending on how it was
produced.
## What changes are included in this PR?
In the `DataType::Struct` branch of `create_hashes_internal!`, when the
struct array has NULLs, the hashes are saved before the fields are
hashed, and the saved value is put back for every NULL row. Without
NULLs the branch works as before and copies nothing.
## What is the testing strategy for this PR?
- New unit test `test_xxhash64_null_struct_ignores_fields`: a struct
whose second row is NULL but whose fields still hold values. The NULL
row keeps the seed and the valid row is hashed.
- `spark/hash/xxhash64.slt` gets the query from the issue, which now
returns `true 42 42`. Without the change it returns
`6158280145716684778` for the masked NULL, so the test fails.
- `cargo test -p datafusion-spark --lib xxhash64` passes, and so do the
`xxhash64` sqllogictest, `cargo fmt --all` and `cargo clippy -p
datafusion-spark --no-deps --all-targets --all-features -- -D warnings`.
## Are there any user-facing changes?
`xxhash64` of a NULL struct now returns the input hash (`42` for a
single argument), matching Spark. There are no API changes.
This PR was written with help from Claude Opus 5.5. I reviewed the
change and ran the tests above myself.
---------
Co-authored-by: mrhard9090 <mo...@gmail.com>
---
datafusion/spark/src/function/hash/utils.rs | 15 +++++++++++-
datafusion/spark/src/function/hash/xxhash64.rs | 28 ++++++++++++++++++++++
.../test_files/spark/hash/xxhash64.slt | 16 +++++++++++++
3 files changed, 58 insertions(+), 1 deletion(-)
diff --git a/datafusion/spark/src/function/hash/utils.rs b/datafusion/spark/src/function/hash/utils.rs
index e7918d33ec..68ee9e8337 100644
--- a/datafusion/spark/src/function/hash/utils.rs
+++ b/datafusion/spark/src/function/hash/utils.rs
@@ -984,7 +984,20 @@ macro_rules! create_hashes_internal {
// Hash each field of the struct - Spark hashes all fields recursively
let columns: Vec<ArrayRef> = struct_array.columns().to_vec();
if !columns.is_empty() {
- $recursive_hash_method(&columns, $hashes_buffer)?;
+ match struct_array.nulls().filter(|nulls| nulls.null_count() > 0) {
+ // A NULL struct leaves the hash unchanged like any other
+ // NULL, whatever its fields still hold.
+ Some(nulls) => {
+ let before = $hashes_buffer.to_vec();
+ $recursive_hash_method(&columns, $hashes_buffer)?;
+ for (i, hash) in $hashes_buffer.iter_mut().enumerate() {
+ if nulls.is_null(i) {
+ *hash = before[i];
+ }
+ }
+ }
+ None => $recursive_hash_method(&columns, $hashes_buffer)?,
+ }
}
}
DataType::Map(field, _) => {
diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rs
index 9d02a51b22..d1d62d6e4a 100644
--- a/datafusion/spark/src/function/hash/xxhash64.rs
+++ b/datafusion/spark/src/function/hash/xxhash64.rs
@@ -423,6 +423,34 @@ mod tests {
assert_ne!(hashes[1], hashes[2]);
}
+ #[test]
+ fn test_xxhash64_null_struct_ignores_fields() {
+ use arrow::array::StructArray;
+ use arrow::buffer::NullBuffer;
+ use arrow::datatypes::{Field, Fields};
+
+ // Row 1 is NULL but its fields still hold values.
+ let fields = Fields::from(vec![
+ Field::new("a", DataType::Int32, false),
+ Field::new("b", DataType::Utf8, false),
+ ]);
+ let struct_array = StructArray::new(
+ fields,
+ vec![
+ Arc::new(Int32Array::from(vec![1, 1])) as ArrayRef,
+ Arc::new(StringArray::from(vec!["x", "x"])) as ArrayRef,
+ ],
+ Some(NullBuffer::from(vec![true, false])),
+ );
+ let array_ref: ArrayRef = Arc::new(struct_array);
+
+ let mut hashes = vec![DEFAULT_SEED; 2];
+ create_xxhash64_hashes(&[array_ref], &mut hashes).unwrap();
+
+ assert_ne!(hashes[0], DEFAULT_SEED);
+ assert_eq!(hashes[1], DEFAULT_SEED);
+ }
+
#[test]
fn test_xxhash64_list() {
use arrow::array::ListArray;
diff --git a/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt
index cbc14967c4..c4b5ad0d59 100644
--- a/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt
+++ b/datafusion/sqllogictest/test_files/spark/hash/xxhash64.slt
@@ -195,3 +195,19 @@ SELECT id, xxhash64(id, name) FROM xxhash_test ORDER BY id;
statement ok
DROP TABLE xxhash_test;
+
+# A NULL struct hashes like any other NULL, whatever its fields still hold
+query BII
+SELECT s IS NULL,
+ xxhash64(s),
+ xxhash64(arrow_cast(NULL, 'Struct("a": Int64, "b": Utf8)'))
+FROM (SELECT nullif(s, s) AS s FROM (SELECT named_struct('a', 1, 'b', 'x') AS s));
+----
+true 42 42
+
+# A NULL struct in a later argument leaves the incoming hash unchanged
+query II
+SELECT xxhash64(7, nullif(s, s)), xxhash64(7)
+FROM (SELECT named_struct('a', 1, 'b', 'x') AS s);
+----
+-2864211556195259663 -2864211556195259663
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org