You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/21 13:10:53 UTC
[doris] branch master updated: [opt](planner) add session var: COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD (#15225)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 56f7ba19c0 [opt](planner) add session var: COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD (#15225)
56f7ba19c0 is described below
commit 56f7ba19c00c30618354732b971e39839c3e8be5
Author: minghong <en...@gmail.com>
AuthorDate: Wed Dec 21 21:10:47 2022 +0800
[opt](planner) add session var: COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD (#15225)
in previous pr(#14876) we compact equals like "a=1 or a=2 or a = 3 " in to "a in (1, 2, 3)"
this pr set a lower bound for the number of equals COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD (default is 2)
for performance reason, we create a hashSet to collect literals, like {1,2,3}. and hence, the literals in "in-predicates" are in random order.
for regression test, if we need stable output of explain string, set COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD to a large number to avoid compact rule.
---
.../src/main/java/org/apache/doris/qe/SessionVariable.java | 14 ++++++++++++++
.../doris/rewrite/CompactEqualsToInPredicateRule.java | 12 +++++++++---
.../data/performance_p0/redundant_conjuncts.out | 2 +-
.../suites/performance_p0/redundant_conjuncts.groovy | 1 +
4 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 5e615850ee..7e22343eb4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -204,6 +204,9 @@ public class SessionVariable implements Serializable, Writable {
// percentage of EXEC_MEM_LIMIT
public static final String BROADCAST_HASHTABLE_MEM_LIMIT_PERCENTAGE = "broadcast_hashtable_mem_limit_percentage";
+
+ public static final String COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD = "compact_equal_to_in_predicate_threshold";
+
public static final String NEREIDS_STAR_SCHEMA_SUPPORT = "nereids_star_schema_support";
public static final String NEREIDS_CBO_PENALTY_FACTOR = "nereids_cbo_penalty_factor";
@@ -550,6 +553,9 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = NEREIDS_STAR_SCHEMA_SUPPORT)
private boolean nereidsStarSchemaSupport = true;
+ @VariableMgr.VarAttr(name = COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD)
+ private int compactEqualToInPredicateThreshold = 2;
+
@VariableMgr.VarAttr(name = NEREIDS_CBO_PENALTY_FACTOR)
private double nereidsCboPenaltyFactor = 0.7;
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TRACE)
@@ -692,6 +698,14 @@ public class SessionVariable implements Serializable, Writable {
this.blockEncryptionMode = blockEncryptionMode;
}
+ public void setCompactEqualToInPredicateThreshold(int threshold) {
+ this.compactEqualToInPredicateThreshold = threshold;
+ }
+
+ public int getCompactEqualToInPredicateThreshold() {
+ return compactEqualToInPredicateThreshold;
+ }
+
public long getMaxExecMemByte() {
return maxExecMemByte;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
index 7375b83121..2b8ad5ed2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
@@ -27,6 +27,7 @@ import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.ExprRewriter.ClauseType;
import com.google.common.collect.Lists;
@@ -44,7 +45,6 @@ a = 1 or a = 2 or a = 3 or a in (4, 5, 6) => a in (1, 2, 3, 4, 5, 6)
*/
public class CompactEqualsToInPredicateRule implements ExprRewriteRule {
public static CompactEqualsToInPredicateRule INSTANCE = new CompactEqualsToInPredicateRule();
- private static final int COMPACT_SIZE = 2;
@Override
public Expr apply(Expr expr, Analyzer analyzer, ClauseType clauseType) throws AnalysisException {
@@ -67,9 +67,15 @@ public class CompactEqualsToInPredicateRule implements ExprRewriteRule {
expr in form of A or B or ...
*/
private Pair<Boolean, Expr> compactEqualsToInPredicate(Expr expr) {
+ int compactThreshold;
+ if (ConnectContext.get() == null) {
+ compactThreshold = 2;
+ } else {
+ compactThreshold = ConnectContext.get().getSessionVariable().getCompactEqualToInPredicateThreshold();
+ }
boolean changed = false;
List<Expr> disConjuncts = getDisconjuncts(expr);
- if (disConjuncts.size() < COMPACT_SIZE) {
+ if (disConjuncts.size() < compactThreshold) {
return Pair.of(false, expr);
}
Map<SlotRef, Set<Expr>> equalMap = new HashMap<>();
@@ -111,7 +117,7 @@ public class CompactEqualsToInPredicateRule implements ExprRewriteRule {
for (Entry<SlotRef, Set<Expr>> entry : equalMap.entrySet()) {
SlotRef slot = entry.getKey();
InPredicate in = inPredMap.get(slot);
- if (entry.getValue().size() >= COMPACT_SIZE || in != null) {
+ if (entry.getValue().size() >= compactThreshold || in != null) {
if (in == null) {
in = new InPredicate(entry.getKey(), Lists.newArrayList(entry.getValue()), false);
inPredMap.put(slot, in);
diff --git a/regression-test/data/performance_p0/redundant_conjuncts.out b/regression-test/data/performance_p0/redundant_conjuncts.out
index 9cba503956..98178f31aa 100644
--- a/regression-test/data/performance_p0/redundant_conjuncts.out
+++ b/regression-test/data/performance_p0/redundant_conjuncts.out
@@ -23,7 +23,7 @@ PLAN FRAGMENT 0
0:VOlapScanNode
TABLE: default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts), PREAGGREGATION: OFF. Reason: No AggregateInfo
- PREDICATES: `k1` IN (2, 1)
+ PREDICATES: (`k1` = 1 OR `k1` = 2)
partitions=0/1, tablets=0/0, tabletList=
cardinality=0, avgRowSize=8.0, numNodes=1
diff --git a/regression-test/suites/performance_p0/redundant_conjuncts.groovy b/regression-test/suites/performance_p0/redundant_conjuncts.groovy
index 3bf5f21309..d42eb03798 100644
--- a/regression-test/suites/performance_p0/redundant_conjuncts.groovy
+++ b/regression-test/suites/performance_p0/redundant_conjuncts.groovy
@@ -35,6 +35,7 @@ suite("redundant_conjuncts") {
EXPLAIN SELECT v1 FROM redundant_conjuncts WHERE k1 = 1 AND k1 = 1;
"""
+ sql "set COMPACT_EQUAL_TO_IN_PREDICATE_THRESHOLD = 100"
qt_redundant_conjuncts_gnerated_by_extract_common_filter """
EXPLAIN SELECT v1 FROM redundant_conjuncts WHERE k1 = 1 OR k1 = 2;
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org