You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/03/09 17:20:39 UTC
[arrow-datafusion] branch master updated: PruningPredicate should take owned Expr (#1960)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new b161216 PruningPredicate should take owned Expr (#1960)
b161216 is described below
commit b1612161e5b2f35c3e69e5594724406a93a093d3
Author: Dan Harris <13...@users.noreply.github.com>
AuthorDate: Wed Mar 9 12:20:32 2022 -0500
PruningPredicate should take owned Expr (#1960)
---
datafusion-jit/src/jit.rs | 6 +++-
datafusion/src/physical_optimizer/pruning.rs | 32 +++++++++++-----------
.../src/physical_plan/file_format/parquet.rs | 19 ++++++-------
3 files changed, 29 insertions(+), 28 deletions(-)
diff --git a/datafusion-jit/src/jit.rs b/datafusion-jit/src/jit.rs
index acce54c..0460cc8 100644
--- a/datafusion-jit/src/jit.rs
+++ b/datafusion-jit/src/jit.rs
@@ -64,7 +64,11 @@ impl Default for JIT {
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {}", msg);
});
- let isa = isa_builder.finish(settings::Flags::new(flag_builder));
+ let isa = isa_builder
+ .finish(settings::Flags::new(flag_builder))
+ .unwrap_or_else(|msg| {
+ panic!("host machine is not supported: {}", msg);
+ });
let builder =
JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
let module = JITModule::new(builder);
diff --git a/datafusion/src/physical_optimizer/pruning.rs b/datafusion/src/physical_optimizer/pruning.rs
index a1263f5..77902c7 100644
--- a/datafusion/src/physical_optimizer/pruning.rs
+++ b/datafusion/src/physical_optimizer/pruning.rs
@@ -121,11 +121,11 @@ impl PruningPredicate {
/// For example, the filter expression `(column / 2) = 4` becomes
/// the pruning predicate
/// `(column_min / 2) <= 4 && 4 <= (column_max / 2))`
- pub fn try_new(expr: &Expr, schema: SchemaRef) -> Result<Self> {
+ pub fn try_new(expr: Expr, schema: SchemaRef) -> Result<Self> {
// build predicate expression once
let mut required_columns = RequiredStatColumns::new();
let logical_predicate_expr =
- build_predicate_expression(expr, schema.as_ref(), &mut required_columns)?;
+ build_predicate_expression(&expr, schema.as_ref(), &mut required_columns)?;
let stat_fields = required_columns
.iter()
.map(|(_, _, f)| f.clone())
@@ -145,7 +145,7 @@ impl PruningPredicate {
schema,
predicate_expr,
required_columns,
- logical_expr: expr.clone(),
+ logical_expr: expr,
})
}
@@ -1358,7 +1358,7 @@ mod tests {
// No stats for s2 ==> some rows could pass
// s2 [3, None] (null max) ==> some rows could pass
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
let expected = vec![false, true, true, true];
@@ -1387,7 +1387,7 @@ mod tests {
// No stats for s2 ==> some rows could pass
// s2 [3, None] (null max) ==> some rows could pass
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
let expected = vec![true, true, true, false, true, true];
assert_eq!(result, expected);
@@ -1431,7 +1431,7 @@ mod tests {
// b1
let expr = col("b1");
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_true);
}
@@ -1442,7 +1442,7 @@ mod tests {
// !b1
let expr = col("b1").not();
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_false);
}
@@ -1453,7 +1453,7 @@ mod tests {
// b1 = true
let expr = col("b1").eq(lit(true));
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_true);
}
@@ -1464,7 +1464,7 @@ mod tests {
// !b1 = true
let expr = col("b1").not().eq(lit(true));
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_false);
}
@@ -1497,13 +1497,13 @@ mod tests {
// i > 0
let expr = col("i").gt(lit(0));
- let p = PruningPredicate::try_new(&expr, schema.clone()).unwrap();
+ let p = PruningPredicate::try_new(expr, schema.clone()).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_ret);
// -i < 0
let expr = Expr::Negative(Box::new(col("i"))).lt(lit(0));
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_ret);
}
@@ -1522,13 +1522,13 @@ mod tests {
// i <= 0
let expr = col("i").lt_eq(lit(0));
- let p = PruningPredicate::try_new(&expr, schema.clone()).unwrap();
+ let p = PruningPredicate::try_new(expr, schema.clone()).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_ret);
// -i >= 0
let expr = Expr::Negative(Box::new(col("i"))).gt_eq(lit(0));
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_ret);
}
@@ -1547,7 +1547,7 @@ mod tests {
// i = 0
let expr = col("i").eq(lit(0));
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_ret);
}
@@ -1566,13 +1566,13 @@ mod tests {
// i > -1
let expr = col("i").gt(lit(-1));
- let p = PruningPredicate::try_new(&expr, schema.clone()).unwrap();
+ let p = PruningPredicate::try_new(expr, schema.clone()).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_ret);
// -i < 1
let expr = Expr::Negative(Box::new(col("i"))).lt(lit(1));
- let p = PruningPredicate::try_new(&expr, schema).unwrap();
+ let p = PruningPredicate::try_new(expr, schema).unwrap();
let result = p.prune(&statistics).unwrap();
assert_eq!(result, expected_ret);
}
diff --git a/datafusion/src/physical_plan/file_format/parquet.rs b/datafusion/src/physical_plan/file_format/parquet.rs
index f585976..2d23ca1 100644
--- a/datafusion/src/physical_plan/file_format/parquet.rs
+++ b/datafusion/src/physical_plan/file_format/parquet.rs
@@ -108,15 +108,12 @@ impl ParquetExec {
let pruning_predicate = predicate.and_then(|predicate_expr| {
match PruningPredicate::try_new(
- &predicate_expr,
+ predicate_expr,
base_config.file_schema.clone(),
) {
Ok(pruning_predicate) => Some(pruning_predicate),
Err(e) => {
- debug!(
- "Could not create pruning predicate for {:?}: {}",
- predicate_expr, e
- );
+ debug!("Could not create pruning predicate for: {}", e);
predicate_creation_errors.add(1);
None
}
@@ -1042,7 +1039,7 @@ mod tests {
// int > 1 => c1_max > 1
let expr = col("c1").gt(lit(15));
let schema = Schema::new(vec![Field::new("c1", DataType::Int32, false)]);
- let pruning_predicate = PruningPredicate::try_new(&expr, Arc::new(schema))?;
+ let pruning_predicate = PruningPredicate::try_new(expr, Arc::new(schema))?;
let schema_descr = get_test_schema_descr(vec![("c1", PhysicalType::INT32)]);
let rgm1 = get_row_group_meta_data(
@@ -1075,7 +1072,7 @@ mod tests {
// int > 1 => c1_max > 1
let expr = col("c1").gt(lit(15));
let schema = Schema::new(vec![Field::new("c1", DataType::Int32, false)]);
- let pruning_predicate = PruningPredicate::try_new(&expr, Arc::new(schema))?;
+ let pruning_predicate = PruningPredicate::try_new(expr, Arc::new(schema))?;
let schema_descr = get_test_schema_descr(vec![("c1", PhysicalType::INT32)]);
let rgm1 = get_row_group_meta_data(
@@ -1114,7 +1111,7 @@ mod tests {
Field::new("c1", DataType::Int32, false),
Field::new("c2", DataType::Int32, false),
]));
- let pruning_predicate = PruningPredicate::try_new(&expr, schema.clone())?;
+ let pruning_predicate = PruningPredicate::try_new(expr, schema.clone())?;
let schema_descr = get_test_schema_descr(vec![
("c1", PhysicalType::INT32),
@@ -1152,7 +1149,7 @@ mod tests {
// if conditions in predicate are joined with OR and an unsupported expression is used
// this bypasses the entire predicate expression and no row groups are filtered out
let expr = col("c1").gt(lit(15)).or(col("c2").modulus(lit(2)));
- let pruning_predicate = PruningPredicate::try_new(&expr, schema)?;
+ let pruning_predicate = PruningPredicate::try_new(expr, schema)?;
let row_group_predicate = build_row_group_predicate(
&pruning_predicate,
parquet_file_metrics(),
@@ -1199,7 +1196,7 @@ mod tests {
Field::new("c1", DataType::Int32, false),
Field::new("c2", DataType::Boolean, false),
]));
- let pruning_predicate = PruningPredicate::try_new(&expr, schema)?;
+ let pruning_predicate = PruningPredicate::try_new(expr, schema)?;
let row_group_metadata = gen_row_group_meta_data_for_pruning_predicate();
let row_group_predicate = build_row_group_predicate(
@@ -1231,7 +1228,7 @@ mod tests {
Field::new("c1", DataType::Int32, false),
Field::new("c2", DataType::Boolean, false),
]));
- let pruning_predicate = PruningPredicate::try_new(&expr, schema)?;
+ let pruning_predicate = PruningPredicate::try_new(expr, schema)?;
let row_group_metadata = gen_row_group_meta_data_for_pruning_predicate();
let row_group_predicate = build_row_group_predicate(