You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/28 11:00:31 UTC
(doris) branch branch-2.1 updated (9e2a9aef7ea -> 240c0416e7d)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from 9e2a9aef7ea [Fix](regression) add disable_auto_compaction for show data (#31519)
new ff120db7a0b [enhance](git)Ignore `fe/gen` (#31512)
new 1a1e043425d [fix](test) fix error messages in fault injection tests (#31515)
new d871852ba93 [fix](Nereids): other cond should be kept for each anti join when expanding anti join such as (#31521)
new ea05b37fb83 Fix analyze mv from follower case bug. (#31523)
new 28b9f5e593f [fix](Nereids) Make the case sensitivity of the result labels compatible with MySQL (#31510)
new 240c0416e7d [nereids] config global partition topn (#31476)
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 1 +
.../nereids/rules/analysis/CheckAfterRewrite.java | 4 +-
.../doris/nereids/rules/analysis/SlotBinder.java | 4 +-
...ogicalPartitionTopNToPhysicalPartitionTopN.java | 70 ++++-
.../doris/nereids/rules/rewrite/OrExpansion.java | 24 +-
.../trees/expressions/ArrayItemReference.java | 10 +
.../trees/expressions/MarkJoinSlotReference.java | 5 +
.../java/org/apache/doris/qe/SessionVariable.java | 13 +
.../data/nereids_p0/union/or_expansion.out | 301 +++++++++++++++++++++
.../limit_push_down/limit_push_down.out | 2 +-
.../push_filter_through_ptopn.out | 2 +-
.../noStatsRfPrune/query67.out | 2 +-
.../no_stats_shape/query67.out | 2 +-
.../test_load_stream_stub_failure_injection.groovy | 2 +-
.../test_multi_replica_fault_injection.groovy | 4 +-
.../explain/test_global_partition_topn_plan.groovy | 76 ++++++
.../suites/nereids_p0/union/or_expansion.groovy | 152 ++++++++---
.../transposeJoin/transposeSemiJoinAgg.groovy | 50 ++--
.../suites/statistics/test_analyze_mtmv.groovy | 37 ++-
.../suites/statistics/test_analyze_mv.groovy | 37 ++-
20 files changed, 692 insertions(+), 106 deletions(-)
create mode 100644 regression-test/suites/nereids_p0/explain/test_global_partition_topn_plan.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/06: [fix](test) fix error messages in fault injection tests (#31515)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1a1e043425d09ed6c1becebefca5ec1cb7bcabad
Author: Kaijie Chen <ck...@apache.org>
AuthorDate: Wed Feb 28 18:06:02 2024 +0800
[fix](test) fix error messages in fault injection tests (#31515)
---
.../fault_injection_p0/test_load_stream_stub_failure_injection.groovy | 2 +-
.../fault_injection_p0/test_multi_replica_fault_injection.groovy | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
index ea790577449..983c755398d 100644
--- a/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
@@ -82,7 +82,7 @@ suite("test_stream_stub_fault_injection", "nonConcurrent") {
}
// StreamSinkFileWriter appendv write segment failed all replica
- load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "failed to write any replicas")
+ load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "failed to send segment data to any replicas")
// StreamSinkFileWriter finalize failed
load_with_injection("StreamSinkFileWriter.finalize.finalize_failed", "failed to finalize any replicas")
// LoadStreams stream wait failed
diff --git a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
index aff1318a52f..37d8b4f2610 100644
--- a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
@@ -93,8 +93,8 @@ suite("test_multi_replica_fault_injection", "nonConcurrent") {
// StreamSinkFileWriter appendv write segment failed two replica
load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", "replica num 1 < load required replica num 2")
// StreamSinkFileWriter appendv write segment failed all replica
- load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "failed to write any replicas")
+ load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "failed to send segment data to any replicas")
sql """ set enable_memtable_on_sink_node=false """
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/06: [fix](Nereids): other cond should be kept for each anti join when expanding anti join such as (#31521)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d871852ba9385ada79c081acc69c9e698d320e9c
Author: 谢健 <ji...@gmail.com>
AuthorDate: Wed Feb 28 18:14:47 2024 +0800
[fix](Nereids): other cond should be kept for each anti join when expanding anti join such as (#31521)
---
.../doris/nereids/rules/rewrite/OrExpansion.java | 24 +-
.../data/nereids_p0/union/or_expansion.out | 301 +++++++++++++++++++++
.../suites/nereids_p0/union/or_expansion.groovy | 152 ++++++++---
3 files changed, 425 insertions(+), 52 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
index 987bfebc403..ff531ffce38 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory;
+import org.apache.doris.nereids.rules.exploration.join.JoinReorderContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.IsNull;
@@ -150,11 +151,11 @@ public class OrExpansion extends OneExplorationRuleFactory {
}
// expand Anti Join:
- // Left Anti join cond1 or cond2 Left Anti join cond1
- // / \ / \
- //left right ===> Anti join cond2 CTERight2
- // / \
- // CTELeft CTERight1
+ // Left Anti join cond1 or cond2, other Left Anti join cond1 and other
+ // / \ / \
+ //left right ===> Anti join cond2 and other CTERight2
+ // / \
+ // CTELeft CTERight1
private Plan expandLeftAntiJoin(CascadesContext ctx,
Pair<List<Expression>, List<Expression>> hashOtherConditions,
LogicalJoin<? extends Plan, ? extends Plan> originJoin,
@@ -171,14 +172,14 @@ public class OrExpansion extends OneExplorationRuleFactory {
replaced.putAll(right.getProducerToConsumerOutputMap());
List<Expression> disjunctions = hashOtherConditions.first;
List<Expression> otherConditions = hashOtherConditions.second;
- otherConditions = otherConditions.stream()
+ List<Expression> newOtherConditions = otherConditions.stream()
.map(e -> e.rewriteUp(s -> replaced.containsKey(s) ? replaced.get(s) : s)).collect(Collectors.toList());
Expression hashCond = disjunctions.get(0);
hashCond = hashCond.rewriteUp(s -> replaced.containsKey(s) ? replaced.get(s) : s);
Plan newPlan = new LogicalJoin<>(JoinType.LEFT_ANTI_JOIN, Lists.newArrayList(hashCond),
- otherConditions, originJoin.getDistributeHint(),
- originJoin.getMarkJoinSlotReference(), left, right, null);
+ newOtherConditions, originJoin.getDistributeHint(),
+ originJoin.getMarkJoinSlotReference(), left, right, JoinReorderContext.EMPTY);
if (hashCond.children().stream().anyMatch(e -> !(e instanceof Slot))) {
Plan normalizedPlan = PushDownExpressionsInHashCondition.pushDownHashExpression(
(LogicalJoin<? extends Plan, ? extends Plan>) newPlan);
@@ -192,10 +193,13 @@ public class OrExpansion extends OneExplorationRuleFactory {
ctx.putCTEIdToConsumer(newRight);
Map<Slot, Slot> newReplaced = new HashMap<>(left.getProducerToConsumerOutputMap());
newReplaced.putAll(newRight.getProducerToConsumerOutputMap());
+ newOtherConditions = otherConditions.stream()
+ .map(e -> e.rewriteUp(s -> newReplaced.containsKey(s) ? newReplaced.get(s) : s))
+ .collect(Collectors.toList());
hashCond = hashCond.rewriteUp(s -> newReplaced.containsKey(s) ? newReplaced.get(s) : s);
newPlan = new LogicalJoin<>(JoinType.LEFT_ANTI_JOIN, Lists.newArrayList(hashCond),
- new ArrayList<>(), originJoin.getDistributeHint(),
- originJoin.getMarkJoinSlotReference(), newPlan, newRight, null);
+ newOtherConditions, originJoin.getDistributeHint(),
+ originJoin.getMarkJoinSlotReference(), newPlan, newRight, JoinReorderContext.EMPTY);
if (hashCond.children().stream().anyMatch(e -> !(e instanceof Slot))) {
newPlan = PushDownExpressionsInHashCondition.pushDownHashExpression(
(LogicalJoin<? extends Plan, ? extends Plan>) newPlan);
diff --git a/regression-test/data/nereids_p0/union/or_expansion.out b/regression-test/data/nereids_p0/union/or_expansion.out
index 4779bf2ace1..3e5df5b3e42 100644
--- a/regression-test/data/nereids_p0/union/or_expansion.out
+++ b/regression-test/data/nereids_p0/union/or_expansion.out
@@ -2,11 +2,18 @@
-- !order_ij --
\N \N
1 \N
+1 \N
+\N 1
+1 1
+1 1
+1 1
+1 1
1 1
1 1
-- !order_laj --
\N
+\N
0
2
3
@@ -20,8 +27,10 @@
-- !order_loj --
\N \N
\N \N
+\N \N
0 \N
1 \N
+1 \N
2 \N
3 \N
4 \N
@@ -30,6 +39,11 @@
7 \N
8 \N
9 \N
+\N 1
+1 1
+1 1
+1 1
+1 1
1 1
1 1
@@ -37,8 +51,89 @@
\N \N
\N \N
\N \N
+\N \N
+\N \N
+0 \N
+1 \N
+1 \N
+2 \N
+3 \N
+4 \N
+5 \N
+6 \N
+7 \N
+8 \N
+9 \N
+\N 1
+1 1
+1 1
+1 1
+1 1
+1 1
+1 1
+\N 20
+\N 21
+\N 22
+\N 23
+\N 24
+\N 25
+\N 26
+\N 27
+\N 28
+\N 29
+
+-- !order_ij_multi_cond --
+\N \N
+1 \N
+1 \N
+\N 1
+1 1
+1 1
+
+-- !order_laj_multi_cond --
+\N
+\N
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+-- !order_loj_multi_cond --
+\N \N
+\N \N
+\N \N
+0 \N
+1 \N
+1 \N
+1 \N
+2 \N
+3 \N
+4 \N
+5 \N
+6 \N
+7 \N
+8 \N
+9 \N
+\N 1
+1 1
+1 1
+
+-- !order_foj_multi_cond --
+\N \N
+\N \N
+\N \N
+\N \N
+\N \N
0 \N
1 \N
+1 \N
+1 \N
2 \N
3 \N
4 \N
@@ -47,6 +142,8 @@
7 \N
8 \N
9 \N
+\N 1
+\N 1
1 1
1 1
\N 20
@@ -60,3 +157,207 @@
\N 28
\N 29
+-- !order_loj_unary_cond --
+\N \N
+\N \N
+\N \N
+0 \N
+1 \N
+1 \N
+1 \N
+2 \N
+3 \N
+4 \N
+5 \N
+6 \N
+7 \N
+8 \N
+9 \N
+\N 1
+1 1
+1 1
+1 1
+1 1
+
+-- !order_foj_unary_cond --
+\N \N
+\N \N
+\N \N
+\N \N
+\N \N
+0 \N
+1 \N
+1 \N
+1 \N
+2 \N
+3 \N
+4 \N
+5 \N
+6 \N
+7 \N
+8 \N
+9 \N
+\N 1
+1 1
+1 1
+1 1
+1 1
+\N 20
+\N 21
+\N 22
+\N 23
+\N 24
+\N 25
+\N 26
+\N 27
+\N 28
+\N 29
+
+-- !order_loj_unary_cond --
+\N \N
+\N \N
+\N \N
+\N \N
+\N \N
+\N \N
+\N \N
+0 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+2 \N
+3 \N
+4 \N
+5 \N
+6 \N
+7 \N
+8 \N
+9 \N
+\N 1
+\N 1
+\N 1
+\N 1
+1 1
+1 1
+1 1
+1 1
+1 1
+1 1
+\N 20
+\N 20
+1 20
+1 20
+\N 21
+\N 21
+1 21
+1 21
+\N 22
+\N 22
+1 22
+1 22
+\N 23
+\N 23
+1 23
+1 23
+\N 24
+\N 24
+1 24
+1 24
+\N 25
+\N 25
+1 25
+1 25
+\N 26
+\N 26
+1 26
+1 26
+\N 27
+\N 27
+1 27
+1 27
+\N 28
+\N 28
+1 28
+1 28
+\N 29
+\N 29
+1 29
+1 29
+
+-- !order_foj_unary_cond --
+\N \N
+\N \N
+\N \N
+\N \N
+\N \N
+\N \N
+\N \N
+0 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+1 \N
+2 \N
+3 \N
+4 \N
+5 \N
+6 \N
+7 \N
+8 \N
+9 \N
+\N 1
+\N 1
+\N 1
+\N 1
+1 1
+1 1
+1 1
+1 1
+1 1
+1 1
+\N 20
+\N 20
+1 20
+1 20
+\N 21
+\N 21
+1 21
+1 21
+\N 22
+\N 22
+1 22
+1 22
+\N 23
+\N 23
+1 23
+1 23
+\N 24
+\N 24
+1 24
+1 24
+\N 25
+\N 25
+1 25
+1 25
+\N 26
+\N 26
+1 26
+1 26
+\N 27
+\N 27
+1 27
+1 27
+\N 28
+\N 28
+1 28
+1 28
+\N 29
+\N 29
+1 29
+1 29
+
diff --git a/regression-test/suites/nereids_p0/union/or_expansion.groovy b/regression-test/suites/nereids_p0/union/or_expansion.groovy
index 2d68098cd2d..eb335b6caeb 100644
--- a/regression-test/suites/nereids_p0/union/or_expansion.groovy
+++ b/regression-test/suites/nereids_p0/union/or_expansion.groovy
@@ -19,12 +19,14 @@ suite("or_expansion") {
sql "SET enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
sql "SET enable_pipeline_engine = true"
+
sql "drop table if exists oe1"
sql "drop table if exists oe2"
sql """
CREATE TABLE IF NOT EXISTS oe1 (
k0 bigint,
- k1 bigint
+ k1 bigint,
+ k2 bigint
)
DUPLICATE KEY(k0)
DISTRIBUTED BY HASH(k0) BUCKETS 1
@@ -36,7 +38,8 @@ suite("or_expansion") {
sql """
CREATE TABLE IF NOT EXISTS oe2 (
k0 bigint,
- k1 bigint
+ k1 bigint,
+ k2 bigint
)
DUPLICATE KEY(k0)
DISTRIBUTED BY HASH(k0) BUCKETS 1
@@ -57,53 +60,63 @@ suite("or_expansion") {
sql """
alter table oe2 modify column k1 set stats ('row_count'='1000', 'ndv'='1000', 'min_value'='1000', 'max_value'='2000', 'avg_size'='1000', 'max_size'='1000' )
"""
+ sql """
+ alter table oe1 modify column k2 set stats ('row_count'='1000', 'ndv'='1000', 'min_value'='1000', 'max_value'='2000', 'avg_size'='1000', 'max_size'='1000' )
+ """
+ sql """
+ alter table oe2 modify column k2 set stats ('row_count'='1000', 'ndv'='1000', 'min_value'='1000', 'max_value'='2000', 'avg_size'='1000', 'max_size'='1000' )
+ """
- explain {
- sql("""
- select oe1.k0, oe2.k0
- from oe1 inner join oe2
- on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
- """)
- contains "VHASH JOIN"
- }
+ // explain {
+ // sql("""
+ // select oe1.k0, oe2.k0
+ // from oe1 inner join oe2
+ // on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
+ // """)
+ // contains "VHASH JOIN"
+ // }
- explain {
- sql("""
- select oe1.k0
- from oe1 left anti join oe2
- on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
- """)
- contains "VHASH JOIN"
- }
+ // explain {
+ // sql("""
+ // select oe1.k0
+ // from oe1 left anti join oe2
+ // on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
+ // """)
+ // contains "VHASH JOIN"
+ // }
- explain {
- sql("""
- select oe1.k0, oe2.k0
- from oe1 left outer join oe2
- on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
- """)
- contains "VHASH JOIN"
- }
+ // explain {
+ // sql("""
+ // select oe1.k0, oe2.k0
+ // from oe1 left outer join oe2
+ // on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
+ // """)
+ // contains "VHASH JOIN"
+ // }
- explain {
- sql("""
- select oe1.k0, oe2.k0
- from oe1 full outer join oe2
- on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
- """)
- contains "VHASH JOIN"
- }
+ // explain {
+ // sql("""
+ // select oe1.k0, oe2.k0
+ // from oe1 full outer join oe2
+ // on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
+ // """)
+ // contains "VHASH JOIN"
+ // }
for (int i = 0; i < 10; i++) {
- sql "insert into oe1 values(${i}, ${i})"
- sql "insert into oe2 values(${i+20}, ${i+20})"
+ sql "insert into oe1 values(${i}, ${i}, ${i})"
+ sql "insert into oe2 values(${i+20}, ${i+20}, ${i+20})"
}
- sql "insert into oe1 values(null, 1)"
- sql "insert into oe1 values(1, null)"
- sql "insert into oe1 values(null, null)"
- sql "insert into oe2 values(null, 1)"
- sql "insert into oe2 values(1, null)"
- sql "insert into oe2 values(null, null)"
+ sql "insert into oe1 values(1, 1, 1)"
+ sql "insert into oe1 values(null, null, null)"
+ sql "insert into oe1 values(null, 1, 1)"
+ sql "insert into oe1 values(1, null, null)"
+ sql "insert into oe1 values(null, null, 1)"
+ sql "insert into oe2 values(1, 1, 1)"
+ sql "insert into oe2 values(null, null, null)"
+ sql "insert into oe2 values(null, 1, 1)"
+ sql "insert into oe2 values(1, null, null)"
+ sql "insert into oe2 values(null, null, 1)"
qt_order_ij """
select oe1.k0, oe2.k0
@@ -132,4 +145,59 @@ suite("or_expansion") {
on oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2
order by oe2.k0, oe1.k0
"""
+
+ qt_order_ij_multi_cond """
+ select oe1.k0, oe2.k0
+ from oe1 inner join oe2
+ on (oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2) and oe1.k2 = oe2.k2
+ order by oe2.k0, oe1.k0
+ """
+
+ qt_order_laj_multi_cond """
+ select oe1.k0
+ from oe1 left anti join oe2
+ on (oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2) and oe1.k2 = oe2.k2
+ order by oe1.k0
+ """
+
+ qt_order_loj_multi_cond """
+ select oe1.k0, oe2.k0
+ from oe1 left outer join oe2
+ on (oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2) and oe1.k2 = oe2.k2
+ order by oe2.k0, oe1.k0
+ """
+
+ qt_order_foj_multi_cond """
+ select oe1.k0, oe2.k0
+ from oe1 full outer join oe2
+ on (oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2) and oe1.k2 = oe2.k2
+ order by oe2.k0, oe1.k0
+ """
+
+ qt_order_loj_unary_cond """
+ select oe1.k0, oe2.k0
+ from oe1 left outer join oe2
+ on (oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2) and oe1.k2 = 1
+ order by oe2.k0, oe1.k0
+ """
+
+ qt_order_foj_unary_cond """
+ select oe1.k0, oe2.k0
+ from oe1 full outer join oe2
+ on (oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2) and oe1.k2 = 1
+ order by oe2.k0, oe1.k0
+ """
+ qt_order_loj_unary_cond """
+ select oe1.k0, oe2.k0
+ from oe1 left outer join oe2
+ on (oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2) or oe1.k2 = 1
+ order by oe2.k0, oe1.k0
+ """
+
+ qt_order_foj_unary_cond """
+ select oe1.k0, oe2.k0
+ from oe1 full outer join oe2
+ on (oe1.k0 = oe2.k0 or oe1.k1 + 1 = oe2.k1 * 2) or oe1.k2 = 1
+ order by oe2.k0, oe1.k0
+ """
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/06: Fix analyze mv from follower case bug. (#31523)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ea05b37fb83156bd8d8468baaf29ea9ba3bd3a65
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed Feb 28 18:32:14 2024 +0800
Fix analyze mv from follower case bug. (#31523)
---
.../suites/statistics/test_analyze_mtmv.groovy | 37 ++++++++++++++++------
.../suites/statistics/test_analyze_mv.groovy | 37 ++++++++++++++++------
2 files changed, 56 insertions(+), 18 deletions(-)
diff --git a/regression-test/suites/statistics/test_analyze_mtmv.groovy b/regression-test/suites/statistics/test_analyze_mtmv.groovy
index d72c718d2aa..a3b49154673 100644
--- a/regression-test/suites/statistics/test_analyze_mtmv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mtmv.groovy
@@ -17,16 +17,35 @@
suite("test_analyze_mtmv") {
- def wait_row_count_reported = { table, expected ->
- for (int i = 0; i < 120; i++) {
- Thread.sleep(5000)
- def result = sql """SHOW DATA FROM ${table};"""
- logger.info("result " + result)
- if (result[0][4] == expected) {
- return;
+ def wait_row_count_reported = { db, table, expected ->
+ def result = sql """show frontends;"""
+ logger.info("show frontends result origin: " + result)
+ def host
+ def port
+ for (int i = 0; i < result.size(); i++) {
+ if (result[i][8] == "true") {
+ host = result[i][1]
+ port = result[i][4]
}
}
- throw new Exception("Row count report timeout.")
+ def tokens = context.config.jdbcUrl.split('/')
+ def url=tokens[0] + "//" + host + ":" + port
+ logger.info("Master url is " + url)
+ connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url) {
+ sql """use ${db}"""
+ result = sql """show frontends;"""
+ logger.info("show frontends result master: " + result)
+ for (int i = 0; i < 120; i++) {
+ Thread.sleep(5000)
+ result = sql """SHOW DATA FROM ${table};"""
+ logger.info("result " + result)
+ if (result[0][4] == expected) {
+ return;
+ }
+ }
+ throw new Exception("Row count report timeout.")
+ }
+
}
sql """drop database if exists test_analyze_mtmv"""
@@ -267,7 +286,7 @@ suite("test_analyze_mtmv") {
result_sample = sql """show column cached stats mv1(sum_total)"""
assertEquals(0, result_sample.size())
- wait_row_count_reported("mv1", "3")
+ wait_row_count_reported("test_analyze_mtmv", "mv1", "3")
sql """analyze table mv1 with sync with sample rows 4000000"""
result_sample = sql """show column stats mv1(l_shipdate)"""
logger.info("result " + result_sample)
diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy
index b433766b55d..5c60c902bf0 100644
--- a/regression-test/suites/statistics/test_analyze_mv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -34,16 +34,35 @@ suite("test_analyze_mv") {
}
}
- def wait_row_count_reported = { table, expected ->
- for (int i = 0; i < 120; i++) {
- Thread.sleep(5000)
- def result = sql """SHOW DATA FROM ${table};"""
- logger.info("result " + result)
- if (result[3][4] == expected) {
- return;
+ def wait_row_count_reported = { db, table, expected ->
+ def result = sql """show frontends;"""
+ logger.info("show frontends result origin: " + result)
+ def host
+ def port
+ for (int i = 0; i < result.size(); i++) {
+ if (result[i][8] == "true") {
+ host = result[i][1]
+ port = result[i][4]
}
}
- throw new Exception("Row count report timeout.")
+ def tokens = context.config.jdbcUrl.split('/')
+ def url=tokens[0] + "//" + host + ":" + port
+ logger.info("Master url is " + url)
+ connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url) {
+ sql """use ${db}"""
+ result = sql """show frontends;"""
+ logger.info("show frontends result master: " + result)
+ for (int i = 0; i < 120; i++) {
+ Thread.sleep(5000)
+ result = sql """SHOW DATA FROM ${table};"""
+ logger.info("result " + result)
+ if (result[3][4] == expected) {
+ return;
+ }
+ }
+ throw new Exception("Row count report timeout.")
+ }
+
}
def wait_analyze_finish = { table ->
@@ -340,7 +359,7 @@ suite("test_analyze_mv") {
assertEquals(0, result_sample.size())
// Test sample
- wait_row_count_reported("mvTestDup", "6")
+ wait_row_count_reported("test_analyze_mv", "mvTestDup", "6")
sql """analyze table mvTestDup with sample rows 4000000"""
wait_analyze_finish("mvTestDup")
result_sample = sql """SHOW ANALYZE mvTestDup;"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 05/06: [fix](Nereids) Make the case sensitivity of the result labels compatible with MySQL (#31510)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 28b9f5e593f3d7c8f1b8a3425bbbb937d3801702
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Wed Feb 28 18:33:42 2024 +0800
[fix](Nereids) Make the case sensitivity of the result labels compatible with MySQL (#31510)
SQL: SELECT iD FROM t1
before the label was: id
after this PR the label will be: iD
---
.../nereids/rules/analysis/CheckAfterRewrite.java | 4 +-
.../doris/nereids/rules/analysis/SlotBinder.java | 4 +-
.../trees/expressions/ArrayItemReference.java | 10 +++++
.../trees/expressions/MarkJoinSlotReference.java | 5 +++
.../transposeJoin/transposeSemiJoinAgg.groovy | 50 +++++++++++-----------
5 files changed, 45 insertions(+), 28 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
index e098e0cac13..b1313a16a19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
@@ -107,9 +107,9 @@ public class CheckAfterRewrite extends OneAnalysisRuleFactory {
.collect(Collectors.toSet());
notFromChildren = removeValidSlotsNotFromChildren(notFromChildren, childrenOutput);
if (!notFromChildren.isEmpty()) {
- if (plan.child(0) instanceof LogicalAggregate) {
+ if (plan.arity() != 0 && plan.child(0) instanceof LogicalAggregate) {
throw new AnalysisException(String.format("%s not in agg's output", notFromChildren
- .stream().map(slot -> slot.getName()).collect(Collectors.joining(", "))));
+ .stream().map(NamedExpression::getName).collect(Collectors.joining(", "))));
} else {
throw new AnalysisException(String.format(
"Input slot(s) not in child's output: %s in plan: %s,"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java
index 7f6e23a120d..8ae79b9c21f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java
@@ -307,7 +307,9 @@ public class SlotBinder extends SubExprAnalyzer {
//TODO: handle name parts more than three.
throw new AnalysisException("Not supported name: "
+ StringUtils.join(nameParts, "."));
- }).collect(Collectors.toList());
+ })
+ .map(s -> s.withName(unboundSlot.getNameParts().get(unboundSlot.getNameParts().size() - 1)))
+ .collect(Collectors.toList());
}
public static boolean compareDbName(String boundedDbName, String unBoundDbName) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
index d2a78859e26..3dd0ef64852 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
@@ -150,6 +150,16 @@ public class ArrayItemReference extends NamedExpression implements ExpectsInputT
return new ArrayItemSlot(exprId, name, dataType, nullable);
}
+ @Override
+ public ArrayItemSlot withName(String name) {
+ return new ArrayItemSlot(exprId, name, dataType, nullable);
+ }
+
+ @Override
+ public SlotReference withNullable(boolean newNullable) {
+ return new ArrayItemSlot(exprId, name, dataType, nullable);
+ }
+
@Override
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitArrayItemSlot(this, context);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java
index 021fcea1a3a..35b217b8a9e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java
@@ -73,4 +73,9 @@ public class MarkJoinSlotReference extends SlotReference implements SlotNotFromC
public MarkJoinSlotReference withExprId(ExprId exprId) {
return new MarkJoinSlotReference(exprId, name, existsHasAgg);
}
+
+ @Override
+ public SlotReference withName(String name) {
+ return new MarkJoinSlotReference(exprId, name, existsHasAgg);
+ }
}
diff --git a/regression-test/suites/nereids_rules_p0/transposeJoin/transposeSemiJoinAgg.groovy b/regression-test/suites/nereids_rules_p0/transposeJoin/transposeSemiJoinAgg.groovy
index c3784219f79..a0e411073fe 100644
--- a/regression-test/suites/nereids_rules_p0/transposeJoin/transposeSemiJoinAgg.groovy
+++ b/regression-test/suites/nereids_rules_p0/transposeJoin/transposeSemiJoinAgg.groovy
@@ -70,47 +70,47 @@ suite("transposeSemiJoinAgg") {
sql "set enable_runtime_filter_prune=false;"
sql '''
- alter table T1 modify column A set stats ('ndv'='5999989709', 'num_nulls'='0', 'row_count'='5999989709');
+ alter table T1 modify column a set stats ('ndv'='5999989709', 'num_nulls'='0', 'row_count'='5999989709');
'''
sql '''
- alter table T1 modify column B set stats ('ndv'='5999989709', 'num_nulls'='0', 'row_count'='5999989709');
+ alter table T1 modify column b set stats ('ndv'='5999989709', 'num_nulls'='0', 'row_count'='5999989709');
'''
sql '''
- alter table T2 modify column A set stats ('ndv'='100', 'num_nulls'='0', 'row_count'='100');
+ alter table T2 modify column a set stats ('ndv'='100', 'num_nulls'='0', 'row_count'='100');
'''
// RULE: TransposeSemiJoinAggProject
// 1. group-by(without grouping sets)
// agg-leftSemi => leftSemi-agg
qt_groupby_positive_case """
explain shape plan
- select T3.A
- from (select A, B, sum(C) from T1 group by A, B) T3
- left semi join T2 on T3.A=T2.A;
+ select T3.a
+ from (select a, b, sum(c) from T1 group by a, b) T3
+ left semi join T2 on T3.a=T2.a;
"""
// agg-leftSemi: agg not pushed down
qt_groupby_negative_case """
explain shape plan
- select T3.A
- from (select A, B, sum(C) as D from T1 group by A, B) T3
- left semi join T2 on T3.D=T2.A;
+ select T3.a
+ from (select a, b, sum(c) as d from T1 group by a, b) T3
+ left semi join T2 on T3.D=T2.a;
"""
// 2 grouping sets
// agg-leftSemi => leftSemi-agg
qt_grouping_positive_case """
explain shape plan
- select T3.A
- from (select A, B, sum(C) from T1 group by grouping sets ((A, B), (A))) T3
- left semi join T2 on T3.A=T2.A;
+ select T3.a
+ from (select a, b, sum(c) from T1 group by grouping sets ((a, b), (a))) T3
+ left semi join T2 on T3.a=T2.a;
"""
// agg-leftSemi: agg not pushed down
qt_grouping_negative_case """
explain shape plan
- select T3.A
- from (select A, B, sum(C) as D from T1 group by grouping sets ((A, B), (A), ())) T3
- left semi join T2 on T3.D=T2.A;
+ select T3.a
+ from (select a, b, sum(c) as D from T1 group by grouping sets ((a, b), (a), ())) T3
+ left semi join T2 on T3.D=T2.a;
"""
// RULE: TransposeSemiJoinAgg
@@ -118,32 +118,32 @@ suite("transposeSemiJoinAgg") {
// agg-leftSemi => leftSemi-agg
qt_groupby_positive_case2 """
explain shape plan
- select T3.A
- from (select A from T1 group by A) T3
- left semi join T2 on T3.A=T2.A;
+ select T3.a
+ from (select a from T1 group by a) T3
+ left semi join T2 on T3.a=T2.a;
"""
// agg-leftSemi: agg not pushed down
qt_groupby_negative_case2 """
explain shape plan
select T3.D
- from (select sum(C) as D from T1 group by A) T3
- left semi join T2 on T3.D=T2.A;
+ from (select sum(c) as D from T1 group by a) T3
+ left semi join T2 on T3.D=T2.a;
"""
// 2 grouping sets
// agg-leftSemi => leftSemi-agg
qt_grouping_positive_case2 """
explain shape plan
- select T3.A
- from (select A from T1 group by grouping sets ((A, B), (A))) T3
- left semi join T2 on T3.A=T2.A;
+ select T3.a
+ from (select a from T1 group by grouping sets ((a, b), (a))) T3
+ left semi join T2 on T3.a=T2.a;
"""
// agg-leftSemi: agg not pushed down
qt_grouping_negative_case2 """
explain shape plan
select T3.D
- from (select sum(C) as D from T1 group by grouping sets ((A, B), (A), ())) T3
- left semi join T2 on T3.D=T2.A;
+ from (select sum(C) as D from T1 group by grouping sets ((a, b), (a), ())) T3
+ left semi join T2 on T3.D=T2.a;
"""
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 06/06: [nereids] config global partition topn (#31476)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 240c0416e7d39239fea989a65823d966ebff7729
Author: xzj7019 <13...@users.noreply.github.com>
AuthorDate: Wed Feb 28 18:47:11 2024 +0800
[nereids] config global partition topn (#31476)
* [nereids] config global partition topn
* [nereids] config global partition topn
---------
Co-authored-by: zhongjian.xzj <zh...@zhongjianxzjdeMacBook-Pro.local>
---
...ogicalPartitionTopNToPhysicalPartitionTopN.java | 70 +++++++++++++++++++-
.../java/org/apache/doris/qe/SessionVariable.java | 13 ++++
.../limit_push_down/limit_push_down.out | 2 +-
.../push_filter_through_ptopn.out | 2 +-
.../noStatsRfPrune/query67.out | 2 +-
.../no_stats_shape/query67.out | 2 +-
.../explain/test_global_partition_topn_plan.groovy | 76 ++++++++++++++++++++++
7 files changed, 162 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPartitionTopNToPhysicalPartitionTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPartitionTopNToPhysicalPartitionTopN.java
index 2b6fcfe464d..410e68f3ec2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPartitionTopNToPhysicalPartitionTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalPartitionTopNToPhysicalPartitionTopN.java
@@ -17,18 +17,26 @@
package org.apache.doris.nereids.rules.implementation;
+import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.OrderExpression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.PartitionTopnPhase;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.ColumnStatistic;
+import org.apache.doris.statistics.Statistics;
import com.google.common.collect.ImmutableList;
import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
/**
* Implementation rule that convert logical partition-top-n to physical partition-top-n.
@@ -42,7 +50,8 @@ public class LogicalPartitionTopNToPhysicalPartitionTopN extends OneImplementati
private List<PhysicalPartitionTopN<? extends Plan>> generatePhysicalPartitionTopn(
LogicalPartitionTopN<? extends Plan> logicalPartitionTopN) {
- if (logicalPartitionTopN.getPartitionKeys().isEmpty()) {
+ if (logicalPartitionTopN.getPartitionKeys().isEmpty()
+ || !checkTwoPhaseGlobalPartitionTopn(logicalPartitionTopN)) {
// if no partition by keys, use local partition topn combined with further full sort
List<OrderKey> orderKeys = !logicalPartitionTopN.getOrderKeys().isEmpty()
? logicalPartitionTopN.getOrderKeys().stream()
@@ -99,6 +108,65 @@ public class LogicalPartitionTopNToPhysicalPartitionTopN extends OneImplementati
}
}
+ /**
+ * check if partition keys' ndv is almost near the total row count.
+ * if yes, it is not suitable for two phase global partition topn.
+ */
+ private boolean checkTwoPhaseGlobalPartitionTopn(LogicalPartitionTopN<? extends Plan> logicalPartitionTopN) {
+ double globalPartitionTopnThreshold = ConnectContext.get().getSessionVariable()
+ .getGlobalPartitionTopNThreshold();
+ if (logicalPartitionTopN.getGroupExpression().isPresent()) {
+ Group group = logicalPartitionTopN.getGroupExpression().get().getOwnerGroup();
+ if (group != null && group.getStatistics() != null) {
+ Statistics stats = group.getStatistics();
+ double rowCount = stats.getRowCount();
+ List<Expression> partitionKeys = logicalPartitionTopN.getPartitionKeys();
+ if (!checkPartitionKeys(partitionKeys)) {
+ return false;
+ }
+ List<ColumnStatistic> partitionByKeyStats = partitionKeys.stream()
+ .map(partitionKey -> stats.findColumnStatistics(partitionKey))
+ .filter(Objects::nonNull)
+ .filter(e -> !e.isUnKnown)
+ .collect(Collectors.toList());
+ if (partitionByKeyStats.size() != partitionKeys.size()) {
+ return false;
+ } else {
+ List<Double> ndvs = partitionByKeyStats.stream().map(s -> s.ndv)
+ .filter(e -> e > 0 && !Double.isInfinite(e))
+ .collect(Collectors.toList());
+ if (ndvs.size() != partitionByKeyStats.size()) {
+ return false;
+ } else {
+ double maxNdv = ndvs.stream().max(Double::compare).get();
+ return rowCount / maxNdv >= globalPartitionTopnThreshold;
+ }
+ }
+ } else {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * global partition topn only take effect if partition keys are columns from basic table
+ */
+ private boolean checkPartitionKeys(List<Expression> partitionKeys) {
+ for (Expression expr : partitionKeys) {
+ if (!(expr instanceof SlotReference)) {
+ return false;
+ } else {
+ SlotReference slot = (SlotReference) expr;
+ if (!slot.getColumn().isPresent() || !slot.getTable().isPresent()) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
private ImmutableList<OrderKey> getAllOrderKeys(LogicalPartitionTopN<? extends Plan> logicalPartitionTopN) {
ImmutableList.Builder<OrderKey> builder = ImmutableList.builder();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 207c48e1f20..ed2f1dcbd4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -209,6 +209,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String MAX_JOIN_NUMBER_BUSHY_TREE = "max_join_number_bushy_tree";
public static final String ENABLE_PARTITION_TOPN = "enable_partition_topn";
+ public static final String GLOBAL_PARTITION_TOPN_THRESHOLD = "global_partition_topn_threshold";
+
public static final String ENABLE_INFER_PREDICATE = "enable_infer_predicate";
public static final long DEFAULT_INSERT_VISIBLE_TIMEOUT_MS = 10_000;
@@ -1027,6 +1029,9 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_PARTITION_TOPN)
private boolean enablePartitionTopN = true;
+ @VariableMgr.VarAttr(name = GLOBAL_PARTITION_TOPN_THRESHOLD)
+ private double globalPartitionTopNThreshold = 100;
+
@VariableMgr.VarAttr(name = ENABLE_INFER_PREDICATE)
private boolean enableInferPredicate = true;
@@ -2533,6 +2538,14 @@ public class SessionVariable implements Serializable, Writable {
this.enablePartitionTopN = enablePartitionTopN;
}
+ public double getGlobalPartitionTopNThreshold() {
+ return globalPartitionTopNThreshold;
+ }
+
+ public void setGlobalPartitionTopnThreshold(int threshold) {
+ this.globalPartitionTopNThreshold = threshold;
+ }
+
public boolean isEnableFoldNondeterministicFn() {
return enableFoldNondeterministicFn;
}
diff --git a/regression-test/data/nereids_rules_p0/limit_push_down/limit_push_down.out b/regression-test/data/nereids_rules_p0/limit_push_down/limit_push_down.out
index 31ffeee07dd..cbf8a09c7e6 100644
--- a/regression-test/data/nereids_rules_p0/limit_push_down/limit_push_down.out
+++ b/regression-test/data/nereids_rules_p0/limit_push_down/limit_push_down.out
@@ -389,7 +389,7 @@ PhysicalResultSink
--PhysicalLimit[GLOBAL]
----PhysicalLimit[LOCAL]
------PhysicalWindow
---------PhysicalPartitionTopN
+--------PhysicalQuickSort[LOCAL_SORT]
----------PhysicalPartitionTopN
------------hashAgg[GLOBAL]
--------------hashAgg[LOCAL]
diff --git a/regression-test/data/nereids_syntax_p0/push_filter_through_ptopn.out b/regression-test/data/nereids_syntax_p0/push_filter_through_ptopn.out
index e37f73ef793..dd23fd8ea50 100644
--- a/regression-test/data/nereids_syntax_p0/push_filter_through_ptopn.out
+++ b/regression-test/data/nereids_syntax_p0/push_filter_through_ptopn.out
@@ -44,7 +44,7 @@ PhysicalResultSink
----PhysicalProject
------filter((T.b = 2) and (rn <= 2))
--------PhysicalWindow
-----------PhysicalPartitionTopN
+----------PhysicalQuickSort[LOCAL_SORT]
------------PhysicalDistribute[DistributionSpecHash]
--------------PhysicalPartitionTopN
----------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query67.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query67.out
index bd3f6458a3c..246d2337283 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query67.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query67.out
@@ -6,7 +6,7 @@ PhysicalResultSink
------PhysicalTopN[LOCAL_SORT]
--------filter((rk <= 100))
----------PhysicalWindow
-------------PhysicalPartitionTopN
+------------PhysicalQuickSort[LOCAL_SORT]
--------------PhysicalDistribute[DistributionSpecHash]
----------------PhysicalPartitionTopN
------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query67.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query67.out
index d069db662b3..900fe97ff05 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query67.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query67.out
@@ -6,7 +6,7 @@ PhysicalResultSink
------PhysicalTopN[LOCAL_SORT]
--------filter((rk <= 100))
----------PhysicalWindow
-------------PhysicalPartitionTopN
+------------PhysicalQuickSort[LOCAL_SORT]
--------------PhysicalDistribute[DistributionSpecHash]
----------------PhysicalPartitionTopN
------------------PhysicalProject
diff --git a/regression-test/suites/nereids_p0/explain/test_global_partition_topn_plan.groovy b/regression-test/suites/nereids_p0/explain/test_global_partition_topn_plan.groovy
new file mode 100644
index 00000000000..7141ac9fce2
--- /dev/null
+++ b/regression-test/suites/nereids_p0/explain/test_global_partition_topn_plan.groovy
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_global_partition_topn_plan") {
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+
+ sql "DROP TABLE IF EXISTS test_global_partition_topn_plan"
+ sql """ CREATE TABLE `test_global_partition_topn_plan` (
+ c1 int, c2 int, c3 int
+ )ENGINE=OLAP
+ distributed by hash(c1) buckets 10
+ properties(
+ "replication_allocation" = "tag.location.default: 1"
+ );"""
+
+ sql """ alter table test_global_partition_topn_plan modify column c1 set stats('row_count'='52899687', 'ndv'='52899687', 'num_nulls'='0', 'min_value'='1', 'max_value'='52899687', 'data_size'='4'); """
+ sql """ alter table test_global_partition_topn_plan modify column c2 set stats('row_count'='52899687', 'ndv'='23622730', 'num_nulls'='0', 'min_value'='1', 'max_value'='52899687', 'data_size'='4'); """
+ sql """ alter table test_global_partition_topn_plan modify column c3 set stats('row_count'='52899687', 'ndv'='2', 'num_nulls'='0', 'min_value'='0', 'max_value'='1', 'data_size'='4'); """
+
+ sql "SET global_partition_topn_threshold=2"
+ explain {
+ sql("shape plan select rn from (select row_number() over (partition by c2 order by c3) as rn from test_global_partition_topn_plan) tmp where rn <= 100");
+ contains"PhysicalPartitionTopN"
+ notContains"PhysicalQuickSort"
+ }
+
+ sql "SET global_partition_topn_threshold=3"
+ explain {
+ sql("shape plan select rn from (select row_number() over (partition by c2 order by c3) as rn from test_global_partition_topn_plan) tmp where rn <= 100");
+ contains"PhysicalPartitionTopN"
+ contains"PhysicalQuickSort"
+ }
+
+ sql "SET global_partition_topn_threshold=100"
+ explain {
+ sql("shape plan select rn from (select row_number() over (partition by c3 order by c2) as rn from test_global_partition_topn_plan) tmp where rn <= 100");
+ contains"PhysicalPartitionTopN"
+ notContains"PhysicalQuickSort"
+ }
+
+ sql "SET global_partition_topn_threshold=2"
+ explain {
+ sql("shape plan select rn from (select row_number() over (partition by c2, c3 order by c1) as rn from test_global_partition_topn_plan) tmp where rn <= 100");
+ contains"PhysicalPartitionTopN"
+ notContains"PhysicalQuickSort"
+ }
+
+ sql "SET global_partition_topn_threshold=3"
+ explain {
+ sql("shape plan select rn from (select row_number() over (partition by c2, c3 order by c1) as rn from test_global_partition_topn_plan) tmp where rn <= 100");
+ contains"PhysicalPartitionTopN"
+ contains"PhysicalQuickSort"
+ }
+
+ sql "SET global_partition_topn_threshold=2"
+ explain {
+ sql("shape plan select rn from (select row_number() over (partition by c2 + c3 order by c1) as rn from test_global_partition_topn_plan) tmp where rn <= 100");
+ contains"PhysicalPartitionTopN"
+ contains"PhysicalQuickSort"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/06: [enhance](git)Ignore `fe/gen` (#31512)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ff120db7a0bdb2d6cf065da0c28f7645ff1cfbcf
Author: zhangdong <49...@qq.com>
AuthorDate: Wed Feb 28 17:59:01 2024 +0800
[enhance](git)Ignore `fe/gen` (#31512)
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index c4d51db53db..79d99ef6d14 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,6 +81,7 @@ samples/doris-demo/remote-udf-cpp-demo/function_server_demo
# FE
fe/mocked/
+fe/gen/
fe/fe-core/gen/
fe/fe-core/src/main/resources/static/
fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.tokens
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org