You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/26 18:49:02 UTC
(datafusion) 01/01: fix: Handle `input_file_name()` pushdown into `ParquetSource` with filter pushdown enabled (#23638)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23638-50ae0762d393549b3052682522e81989e11a76bb
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit dd4dcbc2e425d1e6f547179269af31d3dcc00cd6
Author: Adam Gutglick <ad...@gmail.com>
AuthorDate: Sun Jul 26 19:48:49 2026 +0100
fix: Handle `input_file_name()` pushdown into `ParquetSource` with filter pushdown enabled (#23638)
## Which issue does this PR close?
- Closes #23531.
## Rationale for this change
`input_file_name()` is `FileSource` dependent like `file_row_index()`,
and therefore shouldn't be pushed down into a filter.
## What changes are included in this PR?
`PushdownChecker` now handles both UDFs consistently.
If we keep adding this sort of metadata functions, we might want a
better API to detect them, but for now I think this is a reasonable
approach that isn't very invasive.
## Are these changes tested?
Additional SLT test that verifies that both function behave correctly
with pushdown either enabled or disabled.
## Are there any user-facing changes?
None
---------
Signed-off-by: Adam Gutglick <ad...@gmail.com>
---
.../datasource-parquet/src/projection_read_plan.rs | 5 +++-
.../sqllogictest/test_files/input_file_name.slt | 5 ++--
.../test_files/parquet_metadata_functions.slt | 28 ++++++++++++++++++++++
3 files changed, 34 insertions(+), 4 deletions(-)
diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rs
index c9d8beab14..96c99ab207 100644
--- a/datafusion/datasource-parquet/src/projection_read_plan.rs
+++ b/datafusion/datasource-parquet/src/projection_read_plan.rs
@@ -30,6 +30,7 @@ use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion_functions::core::input_file_name::InputFileNameFunc;
use parquet::arrow::ProjectionMask;
use parquet::schema::types::SchemaDescriptor;
@@ -313,8 +314,10 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
return Ok(recursion);
}
- if ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())
+ if ScalarFunctionExpr::try_downcast_func::<InputFileNameFunc>(node.as_ref())
.is_some()
+ || ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())
+ .is_some()
{
self.has_unpushable_udfs = true;
return Ok(TreeNodeRecursion::Jump);
diff --git a/datafusion/sqllogictest/test_files/input_file_name.slt b/datafusion/sqllogictest/test_files/input_file_name.slt
index 8fb72d4a9d..32110aa2d6 100644
--- a/datafusion/sqllogictest/test_files/input_file_name.slt
+++ b/datafusion/sqllogictest/test_files/input_file_name.slt
@@ -121,8 +121,7 @@ physical_plan
01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST]
02)--SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[true]
03)----FilterExec: __datafusion_extracted_1@0 LIKE %first.parquet, projection=[column1@1]
-04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
-05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet, predicate=input_file_name() LIKE %first.parquet
+04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet
statement ok
-DROP TABLE pq_table;
\ No newline at end of file
+DROP TABLE pq_table;
diff --git a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt
index 773ab6761f..25a3c4eb4c 100644
--- a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt
+++ b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt
@@ -52,5 +52,33 @@ logical_plan
02)--TableScan: test_table projection=[column1]
physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet]]}, projection=[input_file_name() as input_file_name(), CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet
+
+# Make sure it also behaves consistently regardless of filter pushdown
+
+statement ok
+SET datafusion.execution.parquet.pushdown_filters = false;
+
+query TII rowsort
+SELECT input_file_name(), file_row_index(), column1
+FROM test_table
+WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';
+----
+WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30
+WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60
+
+statement ok
+SET datafusion.execution.parquet.pushdown_filters = true;
+
+query TII rowsort
+SELECT input_file_name(), file_row_index(), column1
+FROM test_table
+WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet';
+----
+WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30
+WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60
+
+statement ok
+RESET datafusion.execution.parquet.pushdown_filters;
+
statement ok
DROP TABLE test_table;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org