You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2024/11/11 18:03:34 UTC
(datafusion-comet) 11/14: merge
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch comet-parquet-exec
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit d7396e4f878085b57f3f09ace1e93b549f14a8c8
Merge: 08308407 95d69fa3
Author: Andy Grove <ag...@apache.org>
AuthorDate: Sat Nov 9 09:36:12 2024 -0700
merge
native/core/src/execution/datafusion/planner.rs | 30 ++++++++++++++++---------
1 file changed, 19 insertions(+), 11 deletions(-)
diff --cc native/core/src/execution/datafusion/planner.rs
index 2c7b36ef,c121e93d..c711b03d
--- a/native/core/src/execution/datafusion/planner.rs
+++ b/native/core/src/execution/datafusion/planner.rs
@@@ -1030,21 -1032,18 +1033,26 @@@ impl PhysicalPlanner
.map(|path| PartitionedFile::from_path(path.path().to_string()).unwrap())
.collect();
+ // partition the files
+ // TODO really should partition the row groups
+
+ let mut file_groups = vec![vec![]; partition_count];
+ files.iter().enumerate().for_each(|(idx, file)| {
+ file_groups[partition_count % idx].push(file.clone());
+ });
+
let file_scan_config =
- FileScanConfig::new(object_store_url, data_schema_arrow.clone())
+ FileScanConfig::new(object_store_url, Arc::clone(&data_schema_arrow))
- .with_file_groups(vec![files])
+ .with_file_groups(file_groups)
.with_projection(Some(projection_vector));
- let builder =
- ParquetExecBuilder::new(file_scan_config).with_predicate(test_data_filters);
+ let mut table_parquet_options = TableParquetOptions::new();
+ table_parquet_options.global.pushdown_filters = true;
+ table_parquet_options.global.reorder_filters = true;
+
+ let builder = ParquetExecBuilder::new(file_scan_config)
+ .with_predicate(test_data_filters)
+ .with_table_parquet_options(table_parquet_options);
let scan = builder.build();
return Ok((vec![], Arc::new(scan)));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org