You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ja...@apache.org on 2023/08/01 09:13:07 UTC

[doris] branch master updated: [stats](nereids) fix bug for avg-size (#22421)

This is an automated email from the ASF dual-hosted git repository.

jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new d5d82b7c31 [stats](nereids) fix bug for avg-size (#22421)
d5d82b7c31 is described below

commit d5d82b7c31a93dc9ad4443c35f9d7f3e9d380d1e
Author: minghong <en...@gmail.com>
AuthorDate: Tue Aug 1 17:13:00 2023 +0800

    [stats](nereids) fix bug for avg-size (#22421)
---
 .../doris/nereids/stats/FilterEstimation.java      |  2 +-
 .../apache/doris/statistics/ColumnStatistic.java   |  4 +-
 .../nereids_tpcds_shape_sf100_p0/shape/query65.out | 59 ++++++++++----------
 .../nereids_tpcds_shape_sf100_p0/shape/query72.out | 65 +++++++++++-----------
 .../nereids_tpch_shape_sf1000_p0/shape/q22.out     | 33 ++++++-----
 .../data/nereids_tpch_shape_sf500_p0/shape/q22.out | 33 ++++++-----
 6 files changed, 96 insertions(+), 100 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index 44e9a97a8b..653471e98c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -469,7 +469,7 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
         ColumnStatistic rightColumnStatistic = new ColumnStatisticBuilder(rightStats)
                 .setMinValue(Math.max(leftRange.getLow(), rightRange.getLow()))
                 .setMaxValue(rightRange.getHigh())
-                .setAvgSizeByte(rightStats.ndv * (rightAlwaysGreaterRangeFraction + rightOverlappingRangeFraction))
+                .setNdv(rightStats.ndv * (rightAlwaysGreaterRangeFraction + rightOverlappingRangeFraction))
                 .setNumNulls(0)
                 .build();
         double sel = leftAlwaysLessThanRightPercent
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index 7e23c83e07..bfc5c33302 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -311,8 +311,8 @@ public class ColumnStatistic {
 
     @Override
     public String toString() {
-        return isUnKnown ? "unknown" : String.format("ndv=%.4f, min=%f(%s), max=%f(%s), count=%.4f",
-                ndv, minValue, minExpr, maxValue, maxExpr, count);
+        return isUnKnown ? "unknown" : String.format("ndv=%.4f, min=%f(%s), max=%f(%s), count=%.4f, avgSizeByte=%f",
+                ndv, minValue, minExpr, maxValue, maxExpr, count, avgSizeByte);
     }
 
     public JSONObject toJson() {
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out
index 2d74160159..ae54fc7949 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query65.out
@@ -5,42 +5,41 @@ PhysicalResultSink
 ----PhysicalDistribute
 ------PhysicalTopN
 --------PhysicalProject
-----------hashJoin[INNER_JOIN](item.i_item_sk = sc.ss_item_sk)
-------------PhysicalDistribute
---------------PhysicalProject
-----------------hashJoin[INNER_JOIN](store.s_store_sk = sc.ss_store_sk)
-------------------hashJoin[INNER_JOIN](sb.ss_store_sk = sc.ss_store_sk)(cast(revenue as DOUBLE) <= cast((0.1 * ave) as DOUBLE))
+----------hashJoin[INNER_JOIN](store.s_store_sk = sc.ss_store_sk)
+------------hashJoin[INNER_JOIN](item.i_item_sk = sc.ss_item_sk)
+--------------PhysicalDistribute
+----------------hashJoin[INNER_JOIN](sb.ss_store_sk = sc.ss_store_sk)(cast(revenue as DOUBLE) <= cast((0.1 * ave) as DOUBLE))
+------------------hashAgg[GLOBAL]
+--------------------PhysicalDistribute
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk)
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[store_sales]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------filter((date_dim.d_month_seq <= 1232)(date_dim.d_month_seq >= 1221))
+----------------------------------PhysicalOlapScan[date_dim]
+------------------PhysicalDistribute
 --------------------hashAgg[GLOBAL]
 ----------------------PhysicalDistribute
 ------------------------hashAgg[LOCAL]
 --------------------------PhysicalProject
-----------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk)
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[store_sales]
+----------------------------hashAgg[GLOBAL]
 ------------------------------PhysicalDistribute
---------------------------------PhysicalProject
-----------------------------------filter((date_dim.d_month_seq <= 1232)(date_dim.d_month_seq >= 1221))
-------------------------------------PhysicalOlapScan[date_dim]
---------------------PhysicalDistribute
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------hashAgg[GLOBAL]
---------------------------------PhysicalDistribute
-----------------------------------hashAgg[LOCAL]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk)
+--------------------------------hashAgg[LOCAL]
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN](store_sales.ss_sold_date_sk = date_dim.d_date_sk)
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[store_sales]
+--------------------------------------PhysicalDistribute
 ----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales]
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_month_seq >= 1221)(date_dim.d_month_seq <= 1232))
-----------------------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalDistribute
---------------------PhysicalProject
-----------------------PhysicalOlapScan[store]
+------------------------------------------filter((date_dim.d_month_seq >= 1221)(date_dim.d_month_seq <= 1232))
+--------------------------------------------PhysicalOlapScan[date_dim]
+--------------PhysicalDistribute
+----------------PhysicalProject
+------------------PhysicalOlapScan[item]
 ------------PhysicalDistribute
 --------------PhysicalProject
-----------------PhysicalOlapScan[item]
+----------------PhysicalOlapScan[store]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
index 9571315ac7..90bf3501c5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
@@ -19,42 +19,41 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN](item.i_item_sk = catalog_sales.cs_item_sk)
 --------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN](inventory.inv_date_sk = d2.d_date_sk)(d1.d_week_seq = d2.d_week_seq)
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = inventory.inv_item_sk)(inventory.inv_quantity_on_hand < catalog_sales.cs_quantity)
-----------------------------------------PhysicalDistribute
+----------------------------------PhysicalOlapScan[item]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN](inventory.inv_date_sk = d2.d_date_sk)(d1.d_week_seq = d2.d_week_seq)
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_item_sk = inventory.inv_item_sk)(inventory.inv_quantity_on_hand < catalog_sales.cs_quantity)
 ------------------------------------------PhysicalOlapScan[inventory]
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_ship_date_sk = d3.d_date_sk)(d3.d_date > cast((cast(d_date as BIGINT) + 5) as DATEV2))
-----------------------------------------------PhysicalDistribute
-------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)
---------------------------------------------------PhysicalDistribute
-----------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = d1.d_date_sk)
-------------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)
---------------------------------------------------------PhysicalProject
-----------------------------------------------------------PhysicalOlapScan[catalog_sales]
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_ship_date_sk = d3.d_date_sk)(d3.d_date > cast((cast(d_date as BIGINT) + 5) as DATEV2))
+------------------------------------------------PhysicalDistribute
+--------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)
+----------------------------------------------------PhysicalDistribute
+------------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_sold_date_sk = d1.d_date_sk)
+--------------------------------------------------------hashJoin[INNER_JOIN](catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)
+----------------------------------------------------------PhysicalProject
+------------------------------------------------------------PhysicalOlapScan[catalog_sales]
+----------------------------------------------------------PhysicalDistribute
+------------------------------------------------------------PhysicalProject
+--------------------------------------------------------------filter((cast(hd_buy_potential as VARCHAR(*)) = '501-1000'))
+----------------------------------------------------------------PhysicalOlapScan[household_demographics]
 --------------------------------------------------------PhysicalDistribute
 ----------------------------------------------------------PhysicalProject
-------------------------------------------------------------filter((cast(hd_buy_potential as VARCHAR(*)) = '501-1000'))
---------------------------------------------------------------PhysicalOlapScan[household_demographics]
-------------------------------------------------------PhysicalDistribute
---------------------------------------------------------PhysicalProject
-----------------------------------------------------------filter((d1.d_year = 2002))
-------------------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------------------PhysicalDistribute
-----------------------------------------------------PhysicalProject
-------------------------------------------------------filter((cast(cd_marital_status as VARCHAR(*)) = 'W'))
---------------------------------------------------------PhysicalOlapScan[customer_demographics]
-----------------------------------------------PhysicalDistribute
-------------------------------------------------PhysicalProject
---------------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
---------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[date_dim]
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[item]
+------------------------------------------------------------filter((d1.d_year = 2002))
+--------------------------------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------------------------PhysicalDistribute
+------------------------------------------------------PhysicalProject
+--------------------------------------------------------filter((cast(cd_marital_status as VARCHAR(*)) = 'W'))
+----------------------------------------------------------PhysicalOlapScan[customer_demographics]
+------------------------------------------------PhysicalDistribute
+--------------------------------------------------PhysicalProject
+----------------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------------PhysicalDistribute
+----------------------------------------PhysicalProject
+------------------------------------------PhysicalOlapScan[date_dim]
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[warehouse]
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out
index 3525bb7168..09e3c287f6 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q22.out
@@ -8,22 +8,21 @@ PhysicalResultSink
 ----------PhysicalDistribute
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal))
-------------------PhysicalProject
---------------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey = customer.c_custkey)
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[orders]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
-----------------------------PhysicalOlapScan[customer]
+----------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey = customer.c_custkey)
 ------------------PhysicalDistribute
---------------------PhysicalAssertNumRows
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------filter((customer.c_acctbal > 0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
---------------------------------PhysicalOlapScan[customer]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[orders]
+------------------PhysicalDistribute
+--------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal))
+----------------------PhysicalProject
+------------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
+--------------------------PhysicalOlapScan[customer]
+----------------------PhysicalDistribute
+------------------------PhysicalAssertNumRows
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalDistribute
+------------------------------hashAgg[LOCAL]
+--------------------------------PhysicalProject
+----------------------------------filter((customer.c_acctbal > 0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
+------------------------------------PhysicalOlapScan[customer]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out
index 3525bb7168..09e3c287f6 100644
--- a/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out
+++ b/regression-test/data/nereids_tpch_shape_sf500_p0/shape/q22.out
@@ -8,22 +8,21 @@ PhysicalResultSink
 ----------PhysicalDistribute
 ------------hashAgg[LOCAL]
 --------------PhysicalProject
-----------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal))
-------------------PhysicalProject
---------------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey = customer.c_custkey)
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[orders]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
-----------------------------PhysicalOlapScan[customer]
+----------------hashJoin[RIGHT_ANTI_JOIN](orders.o_custkey = customer.c_custkey)
 ------------------PhysicalDistribute
---------------------PhysicalAssertNumRows
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------filter((customer.c_acctbal > 0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
---------------------------------PhysicalOlapScan[customer]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[orders]
+------------------PhysicalDistribute
+--------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(c_acctbal))
+----------------------PhysicalProject
+------------------------filter(substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
+--------------------------PhysicalOlapScan[customer]
+----------------------PhysicalDistribute
+------------------------PhysicalAssertNumRows
+--------------------------hashAgg[GLOBAL]
+----------------------------PhysicalDistribute
+------------------------------hashAgg[LOCAL]
+--------------------------------PhysicalProject
+----------------------------------filter((customer.c_acctbal > 0.00)substring(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17'))
+------------------------------------PhysicalOlapScan[customer]
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org