You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/23 13:35:14 UTC
(datafusion) 01/01: fix: drop build rows with a dictionary NULL key in null-aware anti joins (#26578)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-26578-ebf48581898530f01d36e5580a29c8b8df3012de
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit a79a694887a17fd5d78882974cd9645d14defbd3
Author: Alejandro Borrego <ab...@gmail.com>
AuthorDate: Fri Oct 23 13:34:48 2026 +0000
fix: drop build rows with a dictionary NULL key in null-aware anti joins (#26578)
## Which issue does this PR close?
- Closes #26568.
## Rationale for this change
`x NOT IN (subquery)` keeps a build row whose key is a dictionary entry
pointing at a NULL value. `NULL NOT IN (5, 6)` is UNKNOWN, so the row
should be dropped, as it is when the key is a physical NULL.
## What changes are included in this PR?
The final step of the uncorrelated null-aware `LeftAnti` join now checks
the logical nulls of the build key instead of `is_null`. They are
computed once with the build side, reusing what #25693 added for
`LeftMark`, so the field and its getter are renamed to
`null_aware_build_key_nulls`.
## What is the testing strategy for this PR?
New `test_null_aware_left_anti_build_logical_null` with the case from
the issue. It fails without the fix.
## Are there any user-facing changes?
No.
Co-authored-by: Alejandro Borrego <21...@users.noreply.github.com>
Co-authored-by: Jay Zhan <ja...@gmail.com>
---
.../physical-plan/src/joins/hash_join/exec.rs | 95 +++++++++++++++++++---
.../physical-plan/src/joins/hash_join/stream.rs | 11 ++-
2 files changed, 91 insertions(+), 15 deletions(-)
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 9f77d782b2..178a55af6e 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -357,11 +357,12 @@ struct JoinBuildData {
/// `build_side_has_null` were computed from the original keys in
/// `collect_left_input`.
values: Vec<ArrayRef>,
- /// Logical nulls of the scalar key for uncorrelated null-aware `LeftMark`.
+ /// Logical nulls of the scalar key for uncorrelated null-aware `LeftMark`
+ /// and `LeftAnti`.
///
/// Computed once while collecting the build side because dictionary
/// logical-null discovery can scan nested dictionary values.
- left_mark_build_key_nulls: Option<NullBuffer>,
+ null_aware_build_key_nulls: Option<NullBuffer>,
/// Bounds computed from the build side; absent for an empty partition.
bounds: Option<PartitionBounds>,
/// IN-list values or a hash-table reference used for filter pushdown.
@@ -438,8 +439,8 @@ impl JoinLeftData {
&self.build.values
}
- pub(super) fn left_mark_build_key_nulls(&self) -> Option<&NullBuffer> {
- self.build.left_mark_build_key_nulls.as_ref()
+ pub(super) fn null_aware_build_key_nulls(&self) -> Option<&NullBuffer> {
+ self.build.null_aware_build_key_nulls.as_ref()
}
/// returns a reference to the visited indices bitmap
@@ -3647,12 +3648,17 @@ fn build_in_memory(
&& !left_values.is_empty()
&& left_values[0].logical_null_count() > 0;
- let left_mark_build_key_nulls = if matches!(
+ let null_aware_build_key_nulls = if matches!(
null_aware,
- Some(NullAwareMode::LeftMark {
- correlated: false,
- ..
- })
+ Some(
+ NullAwareMode::LeftMark {
+ correlated: false,
+ ..
+ } | NullAwareMode::LeftAnti {
+ correlated: false,
+ ..
+ }
+ )
) {
left_values[0]
.logical_nulls()
@@ -3660,7 +3666,7 @@ fn build_in_memory(
} else {
None
};
- if let Some(nulls) = &left_mark_build_key_nulls {
+ if let Some(nulls) = &null_aware_build_key_nulls {
// `logical_nulls` reuses an array's physical null buffer when it can.
// Only charge newly materialized buffers, such as dictionary logical
// nulls, because the build batch's buffers were reserved above.
@@ -3692,7 +3698,7 @@ fn build_in_memory(
map,
batch,
values: normalized_values,
- left_mark_build_key_nulls,
+ null_aware_build_key_nulls,
bounds,
membership,
reservation,
@@ -10549,6 +10555,73 @@ mod tests {
Ok(())
}
+ /// The build side counterpart of
+ /// [`test_null_aware_left_anti_probe_logical_null`]: `NULL NOT IN (5, 6)`
+ /// is UNKNOWN, so the build row whose key points at a NULL dictionary
+ /// value is dropped.
+ #[apply(hash_join_exec_configs)]
+ #[tokio::test]
+ async fn test_null_aware_left_anti_build_logical_null(
+ batch_size: usize,
+ ) -> Result<()> {
+ let task_ctx = prepare_task_ctx(batch_size, false);
+
+ let left = build_table_dict_key(
+ "c1",
+ vec![Some(1), None, Some(2)],
+ vec![0, 1, 2],
+ "dummy",
+ vec![Some(10), Some(0), Some(20)],
+ );
+ let left_key = left
+ .execute(0, Arc::clone(&task_ctx))?
+ .next()
+ .await
+ .unwrap()?;
+ assert_eq!(left_key.column(0).null_count(), 0);
+ assert_eq!(left_key.column(0).logical_null_count(), 1);
+
+ let right = build_table_dict_key(
+ "c2",
+ vec![Some(5), Some(6)],
+ vec![0, 1],
+ "dummy",
+ vec![Some(50), Some(60)],
+ );
+
+ let on = vec![(
+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,
+ )];
+
+ let join = HashJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ &JoinType::LeftAnti,
+ None,
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ true,
+ )?;
+
+ let stream = join.execute(0, task_ctx)?;
+ let batches = common::collect(stream).await?;
+
+ allow_duplicates! {
+ assert_snapshot!(batches_to_sort_string(&batches), @r"
+ +----+-------+
+ | c1 | dummy |
+ +----+-------+
+ | 1 | 10 |
+ | 2 | 20 |
+ +----+-------+
+ ");
+ }
+ Ok(())
+ }
+
/// A dictionary key that points to a NULL dictionary value is a NULL build
/// key even when the dictionary key bitmap itself has no NULLs.
#[apply(hash_join_exec_configs)]
diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs
index db12f65bb0..00506d9c6c 100644
--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs
@@ -1379,12 +1379,15 @@ fn null_aware_left_anti_final_indices(
if !probe_summary.non_empty {
return (left_side, right_side);
}
- // null_aware validation ensures a single join key
- let build_key_column = &left_data.values()[0];
+ // Logical nulls, so that a dictionary key pointing at a NULL value counts
+ // as a NULL build key. Computed once with the build side.
+ let Some(build_key_nulls) = left_data.null_aware_build_key_nulls() else {
+ return (left_side, right_side);
+ };
let left_side = UInt64Array::from_iter_values(left_side.iter().filter_map(|idx| {
let idx =
idx.expect("LeftAnti final indices should always contain build-side rows");
- (!build_key_column.is_null(idx as usize)).then_some(idx)
+ (!build_key_nulls.is_null(idx as usize)).then_some(idx)
}));
let right_side = UInt32Array::new_null(left_side.len());
(left_side, right_side)
@@ -1406,7 +1409,7 @@ fn null_aware_left_mark_column(
let build_key_nulls = if correlated {
None
} else {
- left_data.left_mark_build_key_nulls()
+ left_data.null_aware_build_key_nulls()
};
build_null_aware_left_mark_column(
left_side,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org