You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/25 21:55:06 UTC
(datafusion) branch gh-readonly-queue/main/pr-23848-36c417b6c14244526c81ee5dc4210740ab7b44f8 created (now 6d76482376)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23848-36c417b6c14244526c81ee5dc4210740ab7b44f8
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 6d76482376 fix: Handle null-aware joins correctly in `FilterNullJoinKeys` when its enabled (#23848)
This branch includes the following new commits:
new 6d76482376 fix: Handle null-aware joins correctly in `FilterNullJoinKeys` when its enabled (#23848)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix: Handle null-aware joins correctly in `FilterNullJoinKeys` when its enabled (#23848)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23848-36c417b6c14244526c81ee5dc4210740ab7b44f8
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 6d764823764bb4063d95c82ce0dac385a6cb1fc5
Author: Adam Gutglick <ad...@gmail.com>
AuthorDate: Sat Jul 25 22:54:46 2026 +0100
fix: Handle null-aware joins correctly in `FilterNullJoinKeys` when its enabled (#23848)
## Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->
- Closes #23847 .
## Rationale for this change
Fixes a correctness bug, not sure when that config used/desirable.
This is another thing I ran into during
https://github.com/apache/datafusion/pull/21585
## What changes are included in this PR?
1. New SLT test
2. Fix for bug in `datafusion-optimizer`
## Are these changes tested?
Existing tests, and a new SLT test verifying the config change doesn't
affect the query result and basic unit test.
## Are there any user-facing changes?
None
---------
Signed-off-by: Adam Gutglick <ad...@gmail.com>
---
datafusion/optimizer/src/filter_null_join_keys.rs | 47 ++++++++++++++++++++++
.../test_files/null_aware_anti_join.slt | 14 +++++++
2 files changed, 61 insertions(+)
diff --git a/datafusion/optimizer/src/filter_null_join_keys.rs b/datafusion/optimizer/src/filter_null_join_keys.rs
index c8f419d3e5..e3de8048a8 100644
--- a/datafusion/optimizer/src/filter_null_join_keys.rs
+++ b/datafusion/optimizer/src/filter_null_join_keys.rs
@@ -52,6 +52,7 @@ impl OptimizerRule for FilterNullJoinKeys {
match plan {
LogicalPlan::Join(mut join)
if !join.on.is_empty()
+ && !join.null_aware
&& join.null_equality == NullEquality::NullEqualsNothing =>
{
let (left_preserved, right_preserved) =
@@ -359,4 +360,50 @@ mod tests {
let t2 = table_scan(Some("t2"), &schema, None)?.build()?;
Ok((t1, t2))
}
+
+ #[test]
+ fn null_aware_left_mark_join_keys_not_filtered() -> Result<()> {
+ let (t1, t2) = test_tables()?;
+ let plan = build_null_aware_plan(t1, t2, JoinType::LeftMark)?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ LeftMark Join: t1.id = t2.optional_id null_aware
+ TableScan: t1
+ TableScan: t2
+ ")
+ }
+
+ #[test]
+ fn null_aware_left_anti_join_keys_not_filtered() -> Result<()> {
+ let (t1, t2) = test_tables()?;
+ let plan = build_null_aware_plan(t1, t2, JoinType::LeftAnti)?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ LeftAnti Join: t1.id = t2.optional_id null_aware
+ TableScan: t1
+ TableScan: t2
+ ")
+ }
+
+ /// A join whose nullable right key would get an `IS NOT NULL` filter if it
+ /// were not null-aware.
+ fn build_null_aware_plan(
+ left_table: LogicalPlan,
+ right_table: LogicalPlan,
+ join_type: JoinType,
+ ) -> Result<LogicalPlan> {
+ LogicalPlanBuilder::from(left_table)
+ .join_detailed_with_options(
+ right_table,
+ join_type,
+ (
+ vec![Column::from_qualified_name("t1.id")],
+ vec![Column::from_qualified_name("t2.optional_id")],
+ ),
+ None,
+ NullEquality::NullEqualsNothing,
+ true,
+ )?
+ .build()
+ }
}
diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
index 1d12fc33c9..6061458b13 100644
--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
@@ -70,6 +70,20 @@ query IT rowsort
SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
----
+# Regression test
+
+statement ok
+set datafusion.optimizer.filter_null_join_keys = true;
+
+# The subquery NULL must reach the join: every row's NOT IN is UNKNOWN or
+# FALSE, so the result stays empty.
+query IT rowsort
+SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
+----
+
+statement ok
+reset datafusion.optimizer.filter_null_join_keys;
+
# Verify the result is empty even though there are rows in outer_table
# that don't match the non-NULL value (2) in the subquery.
# This is correct null-aware behavior: if subquery contains NULL, result is unknown.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org