You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/05/11 18:17:47 UTC
(datafusion) branch gh-readonly-queue/main/pr-22084-3d06bedcc9afbd65781ac1de28741c36140d2cbb created (now 7819736952)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-22084-3d06bedcc9afbd65781ac1de28741c36140d2cbb
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 7819736952 Support distinct-from predicates in Parquet pruning (#22084)
This branch includes the following new commits:
new 7819736952 Support distinct-from predicates in Parquet pruning (#22084)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: Support distinct-from predicates in Parquet pruning (#22084)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-22084-3d06bedcc9afbd65781ac1de28741c36140d2cbb
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 781973695205bb419cd57214e46bdc0ce56c5bd9
Author: Daniƫl Heres <da...@gmail.com>
AuthorDate: Mon May 11 20:17:24 2026 +0200
Support distinct-from predicates in Parquet pruning (#22084)
## Which issue does this PR close?
- Closes #.
## Rationale for this change
Parquet statistics pruning did not rewrite `IS DISTINCT FROM` or `IS NOT
DISTINCT FROM`, so row groups that could be proven irrelevant from
min/max and null-count statistics were still kept.
## What changes are included in this PR?
- Adds null-aware pruning rewrites for `IS DISTINCT FROM` and `IS NOT
DISTINCT FROM`.
- Treats distinct-from operators as symmetric when normalizing
scalar-left predicates.
- Refactors shared min/max and null-count pruning expression builders.
- Adds unit tests for pruning predicate evaluation and Parquet row-group
regression coverage.
## Are these changes tested?
## Are there any user-facing changes?
No API changes. Queries using `IS DISTINCT FROM` and `IS NOT DISTINCT
FROM` can now benefit from Parquet statistics pruning.
---
datafusion/core/tests/parquet/row_group_pruning.rs | 75 ++++++
datafusion/expr-common/src/operator.rs | 6 +-
datafusion/pruning/src/pruning_predicate.rs | 289 +++++++++++++++++----
3 files changed, 318 insertions(+), 52 deletions(-)
diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs
index 3ec3541af9..441d1af3e9 100644
--- a/datafusion/core/tests/parquet/row_group_pruning.rs
+++ b/datafusion/core/tests/parquet/row_group_pruning.rs
@@ -1777,6 +1777,15 @@ fn make_i32_batch(
RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from)
}
+fn make_nullable_i32_batch(
+ name: &str,
+ values: Vec<Option<i32>>,
+) -> datafusion_common::error::Result<RecordBatch> {
+ let schema = Arc::new(Schema::new(vec![Field::new(name, DataType::Int32, true)]));
+ let array: ArrayRef = Arc::new(Int32Array::from(values));
+ RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from)
+}
+
// Helper function to create a batch with two Int32 columns
fn make_two_col_i32_batch(
name_a: &str,
@@ -1793,6 +1802,72 @@ fn make_two_col_i32_batch(
RecordBatch::try_new(schema, vec![array_a, array_b]).map_err(DataFusionError::from)
}
+#[tokio::test]
+async fn prune_is_not_distinct_from_i32() -> datafusion_common::error::Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
+ let batches = vec![
+ make_nullable_i32_batch("a", vec![None, None])?,
+ make_nullable_i32_batch("a", vec![Some(0), Some(0)])?,
+ make_nullable_i32_batch("a", vec![Some(0), Some(1)])?,
+ make_nullable_i32_batch("a", vec![Some(2), Some(3)])?,
+ make_nullable_i32_batch("a", vec![None, Some(5)])?,
+ ];
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query("SELECT a FROM t WHERE a IS NOT DISTINCT FROM 0")
+ .with_expected_errors(Some(0))
+ .with_expected_rows(3)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(2))
+ .with_fully_matched_by_stats(Some(1))
+ .with_pruned_by_stats(Some(3))
+ .with_limit_pruned_row_groups(Some(0))
+ .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+ .await;
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query("SELECT a FROM t WHERE a IS NOT DISTINCT FROM NULL")
+ .with_expected_errors(Some(0))
+ .with_expected_rows(3)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(2))
+ .with_fully_matched_by_stats(Some(0))
+ .with_pruned_by_stats(Some(3))
+ .with_limit_pruned_row_groups(Some(0))
+ .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+ .await;
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query("SELECT a FROM t WHERE a IS DISTINCT FROM 0")
+ .with_expected_errors(Some(0))
+ .with_expected_rows(7)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4))
+ .with_fully_matched_by_stats(Some(1))
+ .with_pruned_by_stats(Some(1))
+ .with_limit_pruned_row_groups(Some(0))
+ .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+ .await;
+
+ RowGroupPruningTest::new()
+ .with_scenario(Scenario::Int)
+ .with_query("SELECT a FROM t WHERE a IS DISTINCT FROM NULL")
+ .with_expected_errors(Some(0))
+ .with_expected_rows(7)
+ .with_pruned_files(Some(0))
+ .with_matched_by_stats(Some(4))
+ .with_fully_matched_by_stats(Some(3))
+ .with_pruned_by_stats(Some(1))
+ .with_limit_pruned_row_groups(Some(0))
+ .test_row_group_prune_with_custom_data(schema, batches, 2)
+ .await;
+
+ Ok(())
+}
+
#[tokio::test]
async fn test_limit_pruning_basic() -> datafusion_common::error::Result<()> {
// Scenario: Simple integer column, multiple row groups
diff --git a/datafusion/expr-common/src/operator.rs b/datafusion/expr-common/src/operator.rs
index 427069b326..a078a27f2a 100644
--- a/datafusion/expr-common/src/operator.rs
+++ b/datafusion/expr-common/src/operator.rs
@@ -255,9 +255,9 @@ impl Operator {
Operator::GtEq => Some(Operator::LtEq),
Operator::AtArrow => Some(Operator::ArrowAt),
Operator::ArrowAt => Some(Operator::AtArrow),
- Operator::IsDistinctFrom
- | Operator::IsNotDistinctFrom
- | Operator::Plus
+ Operator::IsDistinctFrom => Some(Operator::IsDistinctFrom),
+ Operator::IsNotDistinctFrom => Some(Operator::IsNotDistinctFrom),
+ Operator::Plus
| Operator::Minus
| Operator::Multiply
| Operator::Divide
diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs
index 76cf14be88..013a06812a 100644
--- a/datafusion/pruning/src/pruning_predicate.rs
+++ b/datafusion/pruning/src/pruning_predicate.rs
@@ -1155,8 +1155,16 @@ fn rewrite_expr_to_prunable(
Ok((left, reverse_operator(op)?, right))
} else if let Some(not) = column_expr.downcast_ref::<phys_expr::NotExpr>() {
// `!col = true` --> `col = !true`
- if op != Operator::Eq && op != Operator::NotEq {
- return plan_err!("Not with operator other than Eq / NotEq is not supported");
+ if !matches!(
+ op,
+ Operator::Eq
+ | Operator::NotEq
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom
+ ) {
+ return plan_err!(
+ "Not with operator other than Eq / NotEq / IsDistinctFrom / IsNotDistinctFrom is not supported"
+ );
}
if not.arg().downcast_ref::<phys_expr::Column>().is_some() {
let left = Arc::clone(not.arg());
@@ -1193,6 +1201,8 @@ fn is_compare_op(op: Operator) -> bool {
| Operator::LtEq
| Operator::Gt
| Operator::GtEq
+ | Operator::IsDistinctFrom
+ | Operator::IsNotDistinctFrom
| Operator::LikeMatch
| Operator::NotLikeMatch
)
@@ -1616,45 +1626,14 @@ fn build_statistics_expr(
expr_builder: &mut PruningExpressionBuilder,
) -> Result<Arc<dyn PhysicalExpr>> {
let statistics_expr: Arc<dyn PhysicalExpr> = match expr_builder.op() {
- Operator::NotEq => {
- // column != literal => (min, max) = literal =>
- // !(min != literal && max != literal) ==>
- // min != literal || literal != max
- let min_column_expr = expr_builder.min_column_expr()?;
- let max_column_expr = expr_builder.max_column_expr()?;
- Arc::new(phys_expr::BinaryExpr::new(
- Arc::new(phys_expr::BinaryExpr::new(
- min_column_expr,
- Operator::NotEq,
- Arc::clone(expr_builder.scalar_expr()),
- )),
- Operator::Or,
- Arc::new(phys_expr::BinaryExpr::new(
- Arc::clone(expr_builder.scalar_expr()),
- Operator::NotEq,
- max_column_expr,
- )),
- ))
- }
+ Operator::NotEq => build_ne_statistics_expr(expr_builder)?,
Operator::Eq => {
// column = literal => (min, max) = literal => min <= literal && literal <= max
// (column / 2) = 4 => (column_min / 2) <= 4 && 4 <= (column_max / 2)
- let min_column_expr = expr_builder.min_column_expr()?;
- let max_column_expr = expr_builder.max_column_expr()?;
- Arc::new(phys_expr::BinaryExpr::new(
- Arc::new(phys_expr::BinaryExpr::new(
- min_column_expr,
- Operator::LtEq,
- Arc::clone(expr_builder.scalar_expr()),
- )),
- Operator::And,
- Arc::new(phys_expr::BinaryExpr::new(
- Arc::clone(expr_builder.scalar_expr()),
- Operator::LtEq,
- max_column_expr,
- )),
- ))
+ build_eq_statistics_expr(expr_builder)?
}
+ Operator::IsDistinctFrom => return build_is_distinct_from(expr_builder),
+ Operator::IsNotDistinctFrom => return build_is_not_distinct_from(expr_builder),
Operator::NotLikeMatch => build_not_like_match(expr_builder)?,
Operator::LikeMatch => build_like_match(expr_builder).ok_or_else(|| {
plan_datafusion_err!(
@@ -1704,6 +1683,126 @@ fn build_statistics_expr(
Ok(statistics_expr)
}
+fn binary_expr(
+ left: Arc<dyn PhysicalExpr>,
+ op: Operator,
+ right: Arc<dyn PhysicalExpr>,
+) -> Arc<dyn PhysicalExpr> {
+ Arc::new(phys_expr::BinaryExpr::new(left, op, right))
+}
+
+fn and_expr(
+ left: Arc<dyn PhysicalExpr>,
+ right: Arc<dyn PhysicalExpr>,
+) -> Arc<dyn PhysicalExpr> {
+ binary_expr(left, Operator::And, right)
+}
+
+fn or_expr(
+ left: Arc<dyn PhysicalExpr>,
+ right: Arc<dyn PhysicalExpr>,
+) -> Arc<dyn PhysicalExpr> {
+ binary_expr(left, Operator::Or, right)
+}
+
+fn build_eq_statistics_expr(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let min_column_expr = expr_builder.min_column_expr()?;
+ let max_column_expr = expr_builder.max_column_expr()?;
+ Ok(and_expr(
+ binary_expr(
+ min_column_expr,
+ Operator::LtEq,
+ Arc::clone(expr_builder.scalar_expr()),
+ ),
+ binary_expr(
+ Arc::clone(expr_builder.scalar_expr()),
+ Operator::LtEq,
+ max_column_expr,
+ ),
+ ))
+}
+
+fn build_ne_statistics_expr(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let min_column_expr = expr_builder.min_column_expr()?;
+ let max_column_expr = expr_builder.max_column_expr()?;
+ Ok(or_expr(
+ binary_expr(
+ min_column_expr,
+ Operator::NotEq,
+ Arc::clone(expr_builder.scalar_expr()),
+ ),
+ binary_expr(
+ Arc::clone(expr_builder.scalar_expr()),
+ Operator::NotEq,
+ max_column_expr,
+ ),
+ ))
+}
+
+fn column_has_nulls_expr(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(binary_expr(
+ expr_builder.null_count_column_expr()?,
+ Operator::Gt,
+ Arc::new(phys_expr::Literal::new(ScalarValue::UInt64(Some(0)))),
+ ))
+}
+
+fn column_has_non_nulls_expr(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ Ok(binary_expr(
+ expr_builder.null_count_column_expr()?,
+ Operator::NotEq,
+ expr_builder.row_count_column_expr()?,
+ ))
+}
+
+fn build_is_distinct_from(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let scalar_expr = Arc::clone(expr_builder.scalar_expr());
+
+ Ok(or_expr(
+ and_expr(
+ Arc::new(phys_expr::IsNullExpr::new(Arc::clone(&scalar_expr))),
+ column_has_non_nulls_expr(expr_builder)?,
+ ),
+ and_expr(
+ Arc::new(phys_expr::IsNotNullExpr::new(scalar_expr)),
+ or_expr(
+ column_has_nulls_expr(expr_builder)?,
+ build_ne_statistics_expr(expr_builder)?,
+ ),
+ ),
+ ))
+}
+
+fn build_is_not_distinct_from(
+ expr_builder: &mut PruningExpressionBuilder,
+) -> Result<Arc<dyn PhysicalExpr>> {
+ let scalar_expr = Arc::clone(expr_builder.scalar_expr());
+
+ Ok(or_expr(
+ and_expr(
+ Arc::new(phys_expr::IsNullExpr::new(Arc::clone(&scalar_expr))),
+ column_has_nulls_expr(expr_builder)?,
+ ),
+ and_expr(
+ Arc::new(phys_expr::IsNotNullExpr::new(scalar_expr)),
+ and_expr(
+ column_has_non_nulls_expr(expr_builder)?,
+ build_eq_statistics_expr(expr_builder)?,
+ ),
+ ),
+ ))
+}
+
/// returns the string literal of the scalar value if it is a string
fn unpack_string(s: &ScalarValue) -> Option<&str> {
s.try_as_str().flatten()
@@ -1920,19 +2019,11 @@ fn wrap_null_count_check_expr(
statistics_expr: Arc<dyn PhysicalExpr>,
expr_builder: &mut PruningExpressionBuilder,
) -> Result<Arc<dyn PhysicalExpr>> {
- // x_null_count != x_row_count
- let not_when_null_count_eq_row_count = Arc::new(phys_expr::BinaryExpr::new(
- expr_builder.null_count_column_expr()?,
- Operator::NotEq,
- expr_builder.row_count_column_expr()?,
- ));
-
// (x_null_count != x_row_count) AND (<statistics_expr>)
- Ok(Arc::new(phys_expr::BinaryExpr::new(
- not_when_null_count_eq_row_count,
- Operator::And,
+ Ok(and_expr(
+ column_has_non_nulls_expr(expr_builder)?,
statistics_expr,
- )))
+ ))
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -1960,7 +2051,7 @@ mod tests {
datatypes::TimeUnit,
};
use datafusion_expr::expr::InList;
- use datafusion_expr::{Expr, cast, is_null, try_cast};
+ use datafusion_expr::{BinaryExpr, Expr, cast, is_null, try_cast};
use datafusion_functions_nested::expr_fn::{array_has, make_array};
use datafusion_physical_expr::expressions::{
self as phys_expr, DynamicFilterPhysicalExpr,
@@ -3992,6 +4083,90 @@ mod tests {
);
}
+ #[test]
+ fn prune_int32_col_is_not_distinct_from() {
+ let (schema, statistics) = int32_setup();
+
+ // Without null counts, IS NOT DISTINCT FROM a non-null literal can
+ // still use min/max ranges, but unknown all-null containers must be kept.
+ let expected_ret = &[true, false, false, true, false];
+
+ prune_with_expr(
+ is_not_distinct_from(col("i"), lit(0)),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ // The operator is symmetric, so the scalar-left form should prune the
+ // same row groups.
+ prune_with_expr(
+ is_not_distinct_from(lit(0), col("i")),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ let statistics = statistics
+ .with_row_counts("i", vec![Some(10), Some(9), None, Some(4), Some(10)])
+ .with_null_counts("i", vec![Some(0), Some(1), None, Some(4), Some(0)]);
+
+ let expected_ret = &[true, false, false, false, false];
+ prune_with_expr(
+ is_not_distinct_from(col("i"), lit(0)),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ let expected_ret = &[false, true, true, true, false];
+ prune_with_expr(
+ is_not_distinct_from(col("i"), lit(ScalarValue::Int32(None))),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+ }
+
+ #[test]
+ fn prune_int32_col_is_distinct_from() {
+ let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, true)]));
+ let statistics = TestStatistics::new().with(
+ "i",
+ ContainerStats::new_i32(
+ vec![Some(0), Some(0), Some(5), None],
+ vec![Some(0), Some(2), Some(5), None],
+ )
+ .with_row_counts(vec![Some(2), Some(2), Some(2), Some(2)])
+ .with_null_counts(vec![Some(0), Some(0), Some(0), Some(2)]),
+ );
+
+ let expected_ret = &[false, true, true, true];
+ prune_with_expr(
+ is_distinct_from(col("i"), lit(0)),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ // The operator is symmetric, so the scalar-left form should prune the
+ // same row groups.
+ prune_with_expr(
+ is_distinct_from(lit(0), col("i")),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+
+ let expected_ret = &[true, true, true, false];
+ prune_with_expr(
+ is_distinct_from(col("i"), lit(ScalarValue::Int32(None))),
+ &schema,
+ &statistics,
+ expected_ret,
+ );
+ }
+
#[test]
fn prune_int32_col_eq_zero_cast() {
let (schema, statistics) = int32_setup();
@@ -5381,6 +5556,22 @@ mod tests {
assert_eq!(result, expected);
}
+ fn is_not_distinct_from(left: Expr, right: Expr) -> Expr {
+ Expr::BinaryExpr(BinaryExpr::new(
+ Box::new(left),
+ Operator::IsNotDistinctFrom,
+ Box::new(right),
+ ))
+ }
+
+ fn is_distinct_from(left: Expr, right: Expr) -> Expr {
+ Expr::BinaryExpr(BinaryExpr::new(
+ Box::new(left),
+ Operator::IsDistinctFrom,
+ Box::new(right),
+ ))
+ }
+
fn test_build_predicate_expression(
expr: &Expr,
schema: &Schema,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org