You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/06/24 22:15:31 UTC

(datafusion) 01/01: Fix projection functional dependency remapping (#23028)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23028-1ed57cb82dadf304a69d30fbddd4da6ed706c22b
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit 12b93675997baafd1f85961efe2cd1b3ddfb0b9e
Author: Huang Qiwei <qi...@qq.com>
AuthorDate: Thu Jun 25 06:15:09 2026 +0800

    Fix projection functional dependency remapping (#23028)
    
    ## Which issue does this PR close?
    
    - Closes #23027.
    
    ## Rationale for this change
    
    Projection functional dependency propagation builds a mapping from
    projected output expressions back to input field indices.
    
    Before this change, projection expressions that are not direct input
    fields, such as planner-generated computed expressions, were omitted
    from that mapping. This shifted the projected positions of later
    passthrough columns. If a primary key column appears after such a
    computed expression, the projected schema can record the primary key
    functional dependency against the wrong output index.
    
    This showed up in the TPC-DS q39 planning path after primary key
    constraints were added to the schemas: downstream aggregate planning was
    reasoning from incorrect functional dependency metadata.
    
    ## What changes are included in this PR?
    
    - Preserves one output slot per projection expression when remapping
    functional dependencies.
    - Uses a sentinel for computed / non-input projection expressions so
    they do not match input functional dependencies, while still keeping
    later passthrough column positions aligned.
    - Applies the same positional behavior to aliases, wildcards, and direct
    projection expressions.
    - Adds a focused regression test for a leading computed projection
    before a primary key column.
    
    ## Are these changes tested?
    
    Yes.
    
    ```text
    cargo fmt --all -- --check
    cargo test -p datafusion-expr projection_with_leading_computed_column_preserves_pk
    cargo test -p datafusion --test tpcds_planning q39
    ```
    
    I also ran a debug SF10 TPC-DS all-query comparison for the fix path
    separately; it completed with 0 failures. I am treating that as
    diagnostic evidence, not a formal benchmark claim.
    
    ## Are there any user-facing changes?
    
    No SQL syntax or public API changes.
    
    Users may see corrected optimized plans and restored performance for
    queries affected by projection functional dependency remapping.
    
    ---------
    
    Co-authored-by: Qiwei Huang <qi...@jsessh.com>
---
 datafusion/expr/src/logical_plan/plan.rs           | 101 +++++++++++++++++++--
 .../test_files/tpch/plans/q17.slt.part             |   4 +-
 2 files changed, 94 insertions(+), 11 deletions(-)

diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index e3c151486f..c154bc7c92 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -4137,8 +4137,12 @@ fn calc_func_dependencies_for_project(
     exprs: &[Expr],
     input: &LogicalPlan,
 ) -> Result<FunctionalDependencies> {
+    // Sentinel for projection outputs that do not map back to any input field.
+    const COMPUTED_EXPR_INDEX: usize = usize::MAX;
+
     let input_fields = input.schema().field_names();
-    // Calculate expression indices (if present) in the input schema.
+    // Map each projection output position to its input column index.
+    // A projection expression can produce multiple output columns, such as `*`.
     let proj_indices = exprs
         .iter()
         .map(|expr| match expr {
@@ -4154,30 +4158,33 @@ fn calc_func_dependencies_for_project(
                 Ok::<_, DataFusionError>(
                     wildcard_fields
                         .into_iter()
-                        .filter_map(|(qualifier, f)| {
+                        .map(|(qualifier, f)| {
                             let flat_name = qualifier
                                 .map(|t| format!("{}.{}", t, f.name()))
                                 .unwrap_or_else(|| f.name().clone());
-                            input_fields.iter().position(|item| *item == flat_name)
+                            input_fields
+                                .iter()
+                                .position(|item| *item == flat_name)
+                                .unwrap_or(COMPUTED_EXPR_INDEX)
                         })
                         .collect::<Vec<_>>(),
                 )
             }
             Expr::Alias(alias) => {
                 let name = format!("{}", alias.expr);
-                Ok(input_fields
+                let input_index = input_fields
                     .iter()
                     .position(|item| *item == name)
-                    .map(|i| vec![i])
-                    .unwrap_or(vec![]))
+                    .unwrap_or(COMPUTED_EXPR_INDEX);
+                Ok(vec![input_index])
             }
             _ => {
                 let name = format!("{expr}");
-                Ok(input_fields
+                let input_index = input_fields
                     .iter()
                     .position(|item| *item == name)
-                    .map(|i| vec![i])
-                    .unwrap_or(vec![]))
+                    .unwrap_or(COMPUTED_EXPR_INDEX);
+                Ok(vec![input_index])
             }
         })
         .collect::<Result<Vec<_>>>()?
@@ -4940,6 +4947,82 @@ mod tests {
         ])
     }
 
+    #[test]
+    fn projection_with_leading_computed_column_preserves_pk() -> Result<()> {
+        let constraints =
+            Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0])]);
+        let source = Arc::new(
+            LogicalTableSource::new(Arc::new(employee_schema()))
+                .with_constraints(constraints),
+        );
+        let plan = LogicalPlanBuilder::scan("employee_csv", source, None)?
+            .project(vec![
+                lit(1i32).alias("__common_expr_1"),
+                col("id"),
+                col("first_name"),
+                col("salary"),
+            ])?
+            .build()?;
+
+        let deps = plan.schema().functional_dependencies();
+        assert_eq!(deps.len(), 1);
+        assert_eq!(deps[0].source_indices, vec![1]);
+
+        Ok(())
+    }
+
+    #[test]
+    fn projection_with_leading_computed_column_and_wildcard_preserves_pk() -> Result<()> {
+        let constraints =
+            Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0])]);
+        let source = Arc::new(
+            LogicalTableSource::new(Arc::new(employee_schema()))
+                .with_constraints(constraints),
+        );
+        let plan = LogicalPlanBuilder::scan("employee_csv", source, None)?
+            .project(vec![
+                SelectExpr::Expression(lit(1i32).alias("__common_expr_1")),
+                SelectExpr::Wildcard(Default::default()),
+            ])?
+            .build()?;
+
+        let deps = plan.schema().functional_dependencies();
+        assert_eq!(plan.schema().fields().len(), 6);
+        assert_eq!(deps.len(), 1);
+        assert_eq!(deps[0].source_indices, vec![1]);
+        assert_eq!(deps[0].target_indices, vec![0, 1, 2, 3, 4, 5]);
+
+        Ok(())
+    }
+
+    #[test]
+    fn projection_with_wildcard_expr_before_pk_preserves_pk() -> Result<()> {
+        let constraints =
+            Constraints::new_unverified(vec![Constraint::PrimaryKey(vec![0])]);
+        let source = Arc::new(
+            LogicalTableSource::new(Arc::new(employee_schema()))
+                .with_constraints(constraints),
+        );
+        let input = LogicalPlanBuilder::scan("employee_csv", source, None)?.build()?;
+        #[expect(deprecated)]
+        let projection = Projection::try_new(
+            vec![
+                Expr::Wildcard {
+                    qualifier: None,
+                    options: Box::new(crate::expr::WildcardOptions::default()),
+                },
+                col("employee_csv.id"),
+            ],
+            Arc::new(input),
+        )?;
+
+        let deps = projection.schema.functional_dependencies();
+        assert_eq!(deps.len(), 1);
+        assert_eq!(deps[0].source_indices, vec![1]);
+
+        Ok(())
+    }
+
     fn i32_split_point(value: i32) -> SplitPoint {
         SplitPoint::new(vec![ScalarValue::Int32(Some(value))])
     }
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q17.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q17.slt.part
index ad23cd9079..e678f8b440 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q17.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q17.slt.part
@@ -39,7 +39,7 @@ logical_plan
 01)Projection: CAST(sum(lineitem.l_extendedprice) AS Float64) / Float64(7) AS avg_yearly
 02)--Aggregate: groupBy=[[]], aggr=[[sum(lineitem.l_extendedprice)]]
 03)----Projection: lineitem.l_extendedprice
-04)------Inner Join: part.p_partkey = __scalar_sq_1.l_partkey Filter: CAST(lineitem.l_quantity AS Decimal128(30, 15)) < __scalar_sq_1.Float64(0.2) * avg(lineitem.l_quantity)
+04)------LeftSemi Join: part.p_partkey = __scalar_sq_1.l_partkey Filter: CAST(lineitem.l_quantity AS Decimal128(30, 15)) < __scalar_sq_1.Float64(0.2) * avg(lineitem.l_quantity)
 05)--------Projection: lineitem.l_quantity, lineitem.l_extendedprice, part.p_partkey
 06)----------Inner Join: lineitem.l_partkey = part.p_partkey
 07)------------TableScan: lineitem projection=[l_partkey, l_quantity, l_extendedprice]
@@ -55,7 +55,7 @@ physical_plan
 02)--AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)]
 03)----CoalescePartitionsExec
 04)------AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)]
-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1]
+05)--------HashJoinExec: mode=Partitioned, join_type=LeftSemi, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1]
 06)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], projection=[l_quantity@1, l_extendedprice@2, p_partkey@3]
 07)------------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=4
 08)--------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_partkey, l_quantity, l_extendedprice], constra [...]


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org