You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by yj...@apache.org on 2023/04/19 01:43:19 UTC
[arrow-datafusion] branch main updated: feat: allow TableScan without projection (#6032)
This is an automated email from the ASF dual-hosted git repository.
yjshen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 5159e61cda feat: allow TableScan without projection (#6032)
5159e61cda is described below
commit 5159e61cdaf5c2e4f81a4e7b1ce2b06ce13ebcb0
Author: Ruihang Xia <wa...@gmail.com>
AuthorDate: Wed Apr 19 09:43:14 2023 +0800
feat: allow TableScan without projection (#6032)
* feat: allow TableScan without projection
Signed-off-by: Ruihang Xia <wa...@gmail.com>
* add test
Signed-off-by: Ruihang Xia <wa...@gmail.com>
* fix clippy
Signed-off-by: Ruihang Xia <wa...@gmail.com>
* Update datafusion/substrait/tests/serialize.rs
Co-authored-by: jakevin <ja...@gmail.com>
---------
Signed-off-by: Ruihang Xia <wa...@gmail.com>
Co-authored-by: jakevin <ja...@gmail.com>
---
datafusion/substrait/src/logical_plan/producer.rs | 58 +++++++++++------------
datafusion/substrait/tests/serialize.rs | 14 ++++++
2 files changed, 41 insertions(+), 31 deletions(-)
diff --git a/datafusion/substrait/src/logical_plan/producer.rs b/datafusion/substrait/src/logical_plan/producer.rs
index f984410624..230221897f 100644
--- a/datafusion/substrait/src/logical_plan/producer.rs
+++ b/datafusion/substrait/src/logical_plan/producer.rs
@@ -118,38 +118,34 @@ pub fn to_substrait_rel(
.collect()
});
- if let Some(struct_items) = projection {
- Ok(Box::new(Rel {
- rel_type: Some(RelType::Read(Box::new(ReadRel {
- common: None,
- base_schema: Some(NamedStruct {
- names: scan
- .source
- .schema()
- .fields()
- .iter()
- .map(|f| f.name().to_owned())
- .collect(),
- r#struct: None,
- }),
- filter: None,
- best_effort_filter: None,
- projection: Some(MaskExpression {
- select: Some(StructSelect { struct_items }),
- maintain_singular_struct: false,
- }),
+ let projection = projection.map(|struct_items| MaskExpression {
+ select: Some(StructSelect { struct_items }),
+ maintain_singular_struct: false,
+ });
+
+ Ok(Box::new(Rel {
+ rel_type: Some(RelType::Read(Box::new(ReadRel {
+ common: None,
+ base_schema: Some(NamedStruct {
+ names: scan
+ .source
+ .schema()
+ .fields()
+ .iter()
+ .map(|f| f.name().to_owned())
+ .collect(),
+ r#struct: None,
+ }),
+ filter: None,
+ best_effort_filter: None,
+ projection,
+ advanced_extension: None,
+ read_type: Some(ReadType::NamedTable(NamedTable {
+ names: scan.table_name.to_vec(),
advanced_extension: None,
- read_type: Some(ReadType::NamedTable(NamedTable {
- names: scan.table_name.to_vec(),
- advanced_extension: None,
- })),
- }))),
- }))
- } else {
- Err(DataFusionError::NotImplemented(
- "TableScan without projection is not supported".to_string(),
- ))
- }
+ })),
+ }))),
+ }))
}
LogicalPlan::Projection(p) => {
let expressions = p
diff --git a/datafusion/substrait/tests/serialize.rs b/datafusion/substrait/tests/serialize.rs
index e58322f79e..a8ea0d7857 100644
--- a/datafusion/substrait/tests/serialize.rs
+++ b/datafusion/substrait/tests/serialize.rs
@@ -17,7 +17,10 @@
#[cfg(test)]
mod tests {
+ use datafusion::datasource::provider_as_source;
+ use datafusion::logical_expr::LogicalPlanBuilder;
use datafusion_substrait::logical_plan::consumer::from_substrait_plan;
+ use datafusion_substrait::logical_plan::producer;
use datafusion_substrait::serializer;
use datafusion::error::Result;
@@ -49,6 +52,17 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn table_scan_without_projection() -> Result<()> {
+ let ctx = create_context().await?;
+ let table = provider_as_source(ctx.table_provider("data").await?);
+ let table_scan = LogicalPlanBuilder::scan("data", table, None)?.build()?;
+ let convert_result = producer::to_substrait_plan(&table_scan);
+ assert!(convert_result.is_ok());
+
+ Ok(())
+ }
+
async fn create_context() -> Result<SessionContext> {
let ctx = SessionContext::new();
ctx.register_csv("data", "tests/testdata/data.csv", CsvReadOptions::new())