(datafusion) 01/01: fix: report a non-nullable return field for named_struct() and struct() (#25306)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25306-2178f275ed51b137b57b92182a485e1f863ffaef
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit 32662e750116bbde2aa2d53929aef7a9f19a361c
Author: Adrian Garcia Badaracco <17...@users.noreply.github.com>
AuthorDate: Tue Sep 15 10:53:57 2026 +0000

    fix: report a non-nullable return field for named_struct() and struct() (#25306)
    
    ## Which issue does this PR close?
    
    - Closes #25305.
    
    ## Rationale for this change
    
    `named_struct(...)` and `struct(...)` never produce a NULL row:
    `invoke_with_args` builds the output `StructArray` with no null buffer.
    Both functions nevertheless reported a nullable return field, and
    `ExprSchemable::nullable` for a scalar function reads that field. As a
    result the simplifier rule that rewrites `a IS NOT NULL` to `true` for a
    non-nullable `a` never fired on a struct constructor.
    
    The user-visible cost is lost pruning. A guard such as `WHERE s IS NOT
    NULL` on a struct built by a view cannot remove any row, but it keeps
    the whole struct expression alive, so projection pushdown can no longer
    prune the scan to the fields that are read:
    
    ```sql
    SET datafusion.explain.format = 'indent';
    CREATE TABLE t (a INT, b INT, c INT) AS VALUES (1, 2, 3), (NULL, 5, 6);
    CREATE VIEW v AS SELECT named_struct('a', a, 'b', b, 'c', c) AS s FROM t;
    EXPLAIN SELECT s['b'] FROM v WHERE s IS NOT NULL;
    ```
    
    Before:
    
    ```
    logical_plan
    Projection: __datafusion_extracted_1 AS v.s[b]
      SubqueryAlias: v
        Projection: __datafusion_extracted_1
          Filter: named_struct(Utf8("a"), t.a, Utf8("b"), t.b, Utf8("c"), t.c) IS NOT NULL
            Projection: t.b AS __datafusion_extracted_1, t.a, t.b, t.c
              TableScan: t projection=[a, b, c]
    ```
    
    After:
    
    ```
    logical_plan
    Projection: __datafusion_extracted_1 AS v.s[b]
      SubqueryAlias: v
        Projection: t.b AS __datafusion_extracted_1
          TableScan: t projection=[b]
    ```
    
    This matches the plan the same query already gets without the guard, and
    lets the existing `get_field(named_struct(...), 'f')` simplification
    (#22239) remove the constructor.
    
    ## What changes are included in this PR?
    
    - `datafusion/functions/src/core/named_struct.rs`:
    `return_field_from_args` now reports a non-nullable field instead of a
    nullable one.
    - `datafusion/functions/src/core/struct.rs`: adds a
    `return_field_from_args` override that reports a non-nullable field. The
    default implementation it previously used made the field nullable.
    
    Both are a one-line nullability correction; the inner struct fields stay
    nullable, since an individual member value can still be NULL.
    
    ## What is the testing strategy for this PR?
    
    - New `sqllogictest` cases at the end of
    `datafusion/sqllogictest/test_files/struct.slt` covering the folded
    scalar results (`named_struct(...) IS NOT NULL`, `struct(...) IS NULL`),
    the `EXPLAIN` for `WHERE s IS NOT NULL` over a view showing `TableScan:
    t projection=[b]`, the values that query returns, and the `CASE WHEN
    named_struct(...) IS NOT NULL` fold.
    - Existing `datafusion-functions` unit tests: `cargo test -p
    datafusion-functions --lib`, 356 passed.
    - The full `sqllogictest` suite: all 518 files pass, with no other
    expectation changes needed.
    
    ## Are there any user-facing changes?
    
    Yes, but no API change.
    
    - The schema nullability of the output of `named_struct()` and
    `struct()` changes from nullable to non-nullable. This now matches the
    array those functions actually build.
    - `IS NULL` and `IS NOT NULL` applied directly to one of these
    constructors constant-fold to `false` and `true`. Query results are
    unchanged; plans get smaller and can prune more.
---
 datafusion/functions/src/core/named_struct.rs |  5 ++-
 datafusion/functions/src/core/struct.rs       | 18 ++++++++--
 datafusion/sqllogictest/test_files/struct.slt | 50 +++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/datafusion/functions/src/core/named_struct.rs b/datafusion/functions/src/core/named_struct.rs
index e1aa441e0b..0d9e133ada 100644
--- a/datafusion/functions/src/core/named_struct.rs
+++ b/datafusion/functions/src/core/named_struct.rs
@@ -141,10 +141,13 @@ impl ScalarUDFImpl for NamedStructFunc {
             .map(|(name, data_type)| Ok(Field::new(name, data_type.to_owned(), true)))
             .collect::<Result<Vec<Field>>>()?;
 
+        // The constructed struct row is never NULL: `invoke_with_args` builds
+        // the `StructArray` without a null buffer, so the output field is
+        // non-nullable and `named_struct(...) IS NOT NULL` folds to `true`.
         Ok(Field::new(
             self.name(),
             DataType::Struct(Fields::from(return_fields)),
-            true,
+            false,
         )
         .into())
     }
diff --git a/datafusion/functions/src/core/struct.rs b/datafusion/functions/src/core/struct.rs
index 2697cb46b0..164b9d2032 100644
--- a/datafusion/functions/src/core/struct.rs
+++ b/datafusion/functions/src/core/struct.rs
@@ -16,9 +16,11 @@
 // under the License.
 
 use arrow::array::StructArray;
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Field, FieldRef};
 use datafusion_common::{Result, exec_err, internal_err};
-use datafusion_expr::{ColumnarValue, Documentation, ScalarFunctionArgs};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
+};
 use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
 use datafusion_macros::user_doc;
 use std::sync::Arc;
@@ -114,6 +116,18 @@ impl ScalarUDFImpl for StructFunc {
         Ok(DataType::Struct(fields))
     }
 
+    fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result<FieldRef> {
+        let arg_types = args
+            .arg_fields
+            .iter()
+            .map(|f| f.data_type().clone())
+            .collect::<Vec<_>>();
+        // The constructed struct row is never NULL: `invoke_with_args` builds
+        // the `StructArray` without a null buffer, so the output field is
+        // non-nullable and `struct(...) IS NOT NULL` folds to `true`.
+        Ok(Field::new(self.name(), self.return_type(&arg_types)?, false).into())
+    }
+
     fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
         let DataType::Struct(fields) = args.return_type() else {
             return internal_err!("incorrect struct return type");
diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt
index 183bfbd04e..7704b7f214 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -1752,3 +1752,53 @@ limit 2;
 
 statement ok
 drop table list_cast_limit;
+
+# named_struct() and struct() never produce a NULL row, so their return field
+# is non-nullable and `IS NOT NULL` on the constructor folds to `true`. A guard
+# on a struct built by a view then no longer keeps the whole struct alive, and
+# the scan is pruned to the fields that are actually read.
+statement ok
+create table struct_ctor_null (a int, b int, c int) as values (1, 2, 3), (NULL, 5, 6);
+
+statement ok
+create view struct_ctor_view as select named_struct('a', a, 'b', b, 'c', c) as s from struct_ctor_null;
+
+query BB
+select named_struct('a', a) is not null, struct(a) is null from struct_ctor_null;
+----
+true false
+true false
+
+query TT
+explain select s['b'] from struct_ctor_view where s is not null;
+----
+logical_plan
+01)Projection: __datafusion_extracted_1 AS struct_ctor_view.s[b]
+02)--SubqueryAlias: struct_ctor_view
+03)----Projection: struct_ctor_null.b AS __datafusion_extracted_1
+04)------TableScan: struct_ctor_null projection=[b]
+physical_plan
+01)ProjectionExec: expr=[b@0 as struct_ctor_view.s[b]]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+select s['b'] from struct_ctor_view where s is not null;
+----
+2
+5
+
+query TT
+explain select case when named_struct('a', a) is not null then b end from struct_ctor_null;
+----
+logical_plan
+01)Projection: struct_ctor_null.b AS CASE WHEN named_struct(Utf8("a"),struct_ctor_null.a) IS NOT NULL THEN struct_ctor_null.b END
+02)--TableScan: struct_ctor_null projection=[b]
+physical_plan
+01)ProjectionExec: expr=[b@0 as CASE WHEN named_struct(Utf8("a"),struct_ctor_null.a) IS NOT NULL THEN struct_ctor_null.b END]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+statement ok
+drop view struct_ctor_view;
+
+statement ok
+drop table struct_ctor_null;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org