You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by al...@apache.org on 2025/04/10 18:43:06 UTC
(datafusion) branch main updated: fix decimal precision issue in simplify expression optimize rule (#15588)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 5fa90dfb78 fix decimal precision issue in simplify expression optimize rule (#15588)
5fa90dfb78 is described below
commit 5fa90dfb7898652ec79ded2e277a71c3787c864a
Author: Jay Zhan <ja...@gmail.com>
AuthorDate: Fri Apr 11 02:43:02 2025 +0800
fix decimal precision issue in simplify expression optimize rule (#15588)
* fix decimal precision
* return expr if fail to find the casted type
* move fn out
* comment
* fmt
* fix
* fix name and doc
---
.../src/simplify_expressions/expr_simplifier.rs | 131 +++++++++++++++++----
.../sqllogictest/test_files/simplify_expr.slt | 32 +++++
2 files changed, 143 insertions(+), 20 deletions(-)
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 9003467703..86c6cbfe66 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -33,8 +33,8 @@ use datafusion_common::{
};
use datafusion_common::{internal_err, DFSchema, DataFusionError, Result, ScalarValue};
use datafusion_expr::{
- and, lit, or, BinaryExpr, Case, ColumnarValue, Expr, Like, Operator, Volatility,
- WindowFunctionDefinition,
+ and, binary::BinaryTypeCoercer, lit, or, BinaryExpr, Case, ColumnarValue, Expr, Like,
+ Operator, Volatility, WindowFunctionDefinition,
};
use datafusion_expr::{expr::ScalarFunction, interval_arithmetic::NullableInterval};
use datafusion_expr::{
@@ -976,30 +976,39 @@ impl<S: SimplifyInfo> TreeNodeRewriter for Simplifier<'_, S> {
// Rules for Multiply
//
- // A * 1 --> A
+ // A * 1 --> A (with type coercion if needed)
Expr::BinaryExpr(BinaryExpr {
left,
op: Multiply,
right,
- }) if is_one(&right) => Transformed::yes(*left),
- // 1 * A --> A
+ }) if is_one(&right) => {
+ simplify_right_is_one_case(info, left, &Multiply, &right)?
+ }
+ // A * null --> null
Expr::BinaryExpr(BinaryExpr {
left,
op: Multiply,
right,
- }) if is_one(&left) => Transformed::yes(*right),
- // A * null --> null
+ }) if is_null(&right) => {
+ simplify_right_is_null_case(info, &left, &Multiply, right)?
+ }
+ // 1 * A --> A
Expr::BinaryExpr(BinaryExpr {
- left: _,
+ left,
op: Multiply,
right,
- }) if is_null(&right) => Transformed::yes(*right),
+ }) if is_one(&left) => {
+ // 1 * A is equivalent to A * 1
+ simplify_right_is_one_case(info, right, &Multiply, &left)?
+ }
// null * A --> null
Expr::BinaryExpr(BinaryExpr {
left,
op: Multiply,
- right: _,
- }) if is_null(&left) => Transformed::yes(*left),
+ right,
+ }) if is_null(&left) => {
+ simplify_right_is_null_case(info, &right, &Multiply, left)?
+ }
// A * 0 --> 0 (if A is not null and not floating, since NAN * 0 -> NAN)
Expr::BinaryExpr(BinaryExpr {
@@ -1033,19 +1042,23 @@ impl<S: SimplifyInfo> TreeNodeRewriter for Simplifier<'_, S> {
left,
op: Divide,
right,
- }) if is_one(&right) => Transformed::yes(*left),
- // null / A --> null
+ }) if is_one(&right) => {
+ simplify_right_is_one_case(info, left, &Divide, &right)?
+ }
+ // A / null --> null
Expr::BinaryExpr(BinaryExpr {
left,
op: Divide,
- right: _,
- }) if is_null(&left) => Transformed::yes(*left),
- // A / null --> null
+ right,
+ }) if is_null(&right) => {
+ simplify_right_is_null_case(info, &left, &Divide, right)?
+ }
+ // null / A --> null
Expr::BinaryExpr(BinaryExpr {
- left: _,
+ left,
op: Divide,
right,
- }) if is_null(&right) => Transformed::yes(*right),
+ }) if is_null(&left) => simplify_null_div_other_case(info, left, &right)?,
//
// Rules for Modulo
@@ -1997,6 +2010,84 @@ fn is_exactly_true(expr: Expr, info: &impl SimplifyInfo) -> Result<Expr> {
}
}
+// A * 1 -> A
+// A / 1 -> A
+//
+// Move this function body out of the large match branch avoid stack overflow
+fn simplify_right_is_one_case<S: SimplifyInfo>(
+ info: &S,
+ left: Box<Expr>,
+ op: &Operator,
+ right: &Expr,
+) -> Result<Transformed<Expr>> {
+ // Check if resulting type would be different due to coercion
+ let left_type = info.get_data_type(&left)?;
+ let right_type = info.get_data_type(right)?;
+ match BinaryTypeCoercer::new(&left_type, op, &right_type).get_result_type() {
+ Ok(result_type) => {
+ // Only cast if the types differ
+ if left_type != result_type {
+ Ok(Transformed::yes(Expr::Cast(Cast::new(left, result_type))))
+ } else {
+ Ok(Transformed::yes(*left))
+ }
+ }
+ Err(_) => Ok(Transformed::yes(*left)),
+ }
+}
+
+// A * null -> null
+// A / null -> null
+//
+// Move this function body out of the large match branch avoid stack overflow
+fn simplify_right_is_null_case<S: SimplifyInfo>(
+ info: &S,
+ left: &Expr,
+ op: &Operator,
+ right: Box<Expr>,
+) -> Result<Transformed<Expr>> {
+ // Check if resulting type would be different due to coercion
+ let left_type = info.get_data_type(left)?;
+ let right_type = info.get_data_type(&right)?;
+ match BinaryTypeCoercer::new(&left_type, op, &right_type).get_result_type() {
+ Ok(result_type) => {
+ // Only cast if the types differ
+ if right_type != result_type {
+ Ok(Transformed::yes(Expr::Cast(Cast::new(right, result_type))))
+ } else {
+ Ok(Transformed::yes(*right))
+ }
+ }
+ Err(_) => Ok(Transformed::yes(*right)),
+ }
+}
+
+// null / A --> null
+//
+// Move this function body out of the large match branch avoid stack overflow
+fn simplify_null_div_other_case<S: SimplifyInfo>(
+ info: &S,
+ left: Box<Expr>,
+ right: &Expr,
+) -> Result<Transformed<Expr>> {
+ // Check if resulting type would be different due to coercion
+ let left_type = info.get_data_type(&left)?;
+ let right_type = info.get_data_type(right)?;
+ match BinaryTypeCoercer::new(&left_type, &Operator::Divide, &right_type)
+ .get_result_type()
+ {
+ Ok(result_type) => {
+ // Only cast if the types differ
+ if left_type != result_type {
+ Ok(Transformed::yes(Expr::Cast(Cast::new(left, result_type))))
+ } else {
+ Ok(Transformed::yes(*left))
+ }
+ }
+ Err(_) => Ok(Transformed::yes(*left)),
+ }
+}
+
#[cfg(test)]
mod tests {
use crate::simplify_expressions::SimplifyContext;
@@ -2316,12 +2407,12 @@ mod tests {
// A / null --> null
let null = lit(ScalarValue::Null);
{
- let expr = col("c") / null.clone();
+ let expr = col("c1") / null.clone();
assert_eq!(simplify(expr), null);
}
// null / A --> null
{
- let expr = null.clone() / col("c");
+ let expr = null.clone() / col("c1");
assert_eq!(simplify(expr), null);
}
}
diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.slt
index 43193fb41c..ac4b5b0395 100644
--- a/datafusion/sqllogictest/test_files/simplify_expr.slt
+++ b/datafusion/sqllogictest/test_files/simplify_expr.slt
@@ -65,3 +65,35 @@ select b from t where b !~ '.*'
statement ok
drop table t;
+
+# test decimal precision
+query B
+SELECT a * 1.000::DECIMAL(4,3) > 1.2::decimal(2,1) FROM VALUES (1) AS t(a);
+----
+false
+
+query B
+SELECT 1.000::DECIMAL(4,3) * a > 1.2::decimal(2,1) FROM VALUES (1) AS t(a);
+----
+false
+
+query B
+SELECT NULL::DECIMAL(4,3) * a > 1.2::decimal(2,1) FROM VALUES (1) AS t(a);
+----
+NULL
+
+query B
+SELECT a * NULL::DECIMAL(4,3) > 1.2::decimal(2,1) FROM VALUES (1) AS t(a);
+----
+NULL
+
+query B
+SELECT a / 1.000::DECIMAL(4,3) > 1.2::decimal(2,1) FROM VALUES (1) AS t(a);
+----
+false
+
+query B
+SELECT a / NULL::DECIMAL(4,3) > 1.2::decimal(2,1) FROM VALUES (1) AS t(a);
+----
+NULL
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org