You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2020/04/02 01:11:53 UTC
[incubator-doris] branch master updated: Rewrite count distinct
bitmap and hll in order by and having (#3232)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6252a27 Rewrite count distinct bitmap and hll in order by and having (#3232)
6252a27 is described below
commit 6252a271ddaa90865abef839022a411ed8218007
Author: kangkaisen <ka...@apache.org>
AuthorDate: Thu Apr 2 09:11:42 2020 +0800
Rewrite count distinct bitmap and hll in order by and having (#3232)
---
.../java/org/apache/doris/analysis/QueryStmt.java | 38 +++++++++++++++++++++-
.../java/org/apache/doris/analysis/SelectStmt.java | 32 +-----------------
.../org/apache/doris/planner/QueryPlanTest.java | 8 +++++
3 files changed, 46 insertions(+), 32 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java
index bc1bed8..1c96f45 100644
--- a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java
@@ -18,6 +18,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@@ -27,6 +28,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.doris.qe.ConnectContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -202,6 +204,38 @@ public abstract class QueryStmt extends StatementBase {
this.needToSql = needToSql;
}
+
+ // for bitmap type, we rewrite count distinct to bitmap_union_count
+ // for hll type, we rewrite count distinct to hll_union_agg
+ protected Expr rewriteCountDistinctForBitmapOrHLL(Expr expr, Analyzer analyzer) {
+ if (ConnectContext.get() == null || !ConnectContext.get().getSessionVariable().isRewriteCountDistinct()) {
+ return expr;
+ }
+
+ for (int i = 0; i < expr.getChildren().size(); ++i) {
+ expr.setChild(i, rewriteCountDistinctForBitmapOrHLL(expr.getChild(i), analyzer));
+ }
+
+ if (!(expr instanceof FunctionCallExpr)) {
+ return expr;
+ }
+
+ FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr;
+ if (functionCallExpr.isCountDistinctBitmapOrHLL()) {
+ FunctionParams newParams = new FunctionParams(false, functionCallExpr.getParams().exprs());
+ if (expr.getChild(0).type.isBitmapType()) {
+ FunctionCallExpr bitmapExpr = new FunctionCallExpr(FunctionSet.BITMAP_UNION_COUNT, newParams);
+ bitmapExpr.analyzeNoThrow(analyzer);
+ return bitmapExpr;
+ } else {
+ FunctionCallExpr hllExpr = new FunctionCallExpr("hll_union_agg", newParams);
+ hllExpr.analyzeNoThrow(analyzer);
+ return hllExpr;
+ }
+ }
+ return expr;
+ }
+
/**
* Creates sortInfo by resolving aliases and ordinals in the orderingExprs.
* If the query stmt is an inline view/union operand, then order-by with no
@@ -234,7 +268,9 @@ public abstract class QueryStmt extends StatementBase {
// save the order by element after analyzed
orderByElementsAfterAnalyzed = Lists.newArrayList();
for (int i = 0; i < orderByElements.size(); i++) {
- OrderByElement orderByElement = new OrderByElement(orderingExprs.get(i), isAscOrder.get(i),
+ // rewrite count distinct
+ Expr rewritten = rewriteCountDistinctForBitmapOrHLL(orderingExprs.get(i), analyzer);
+ OrderByElement orderByElement = new OrderByElement(rewritten, isAscOrder.get(i),
nullsFirstParams.get(i));
orderByElementsAfterAnalyzed.add(orderByElement);
}
diff --git a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 0675bee..ffa5c01 100644
--- a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -21,7 +21,6 @@ import org.apache.doris.catalog.AggregateFunction;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
-import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table.TableType;
import org.apache.doris.catalog.Type;
@@ -791,36 +790,6 @@ public class SelectStmt extends QueryStmt {
}
}
- // for bitmap type, we rewrite count distinct to bitmap_union_count
- // for hll type, we rewrite count distinct to hll_union_agg
- private Expr rewriteCountDistinctForBitmapOrHLL(Expr expr, Analyzer analyzer) {
- if (ConnectContext.get() == null || !ConnectContext.get().getSessionVariable().isRewriteCountDistinct()) {
- return expr;
- }
-
- for (int i = 0; i < expr.getChildren().size(); ++i) {
- expr.setChild(i, rewriteCountDistinctForBitmapOrHLL(expr.getChild(i), analyzer));
- }
-
- if (!(expr instanceof FunctionCallExpr)) {
- return expr;
- }
-
- FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr;
- if (functionCallExpr.isCountDistinctBitmapOrHLL()) {
- FunctionParams newParams = new FunctionParams(false, functionCallExpr.getParams().exprs());
- if (expr.getChild(0).type.isBitmapType()) {
- FunctionCallExpr bitmapExpr = new FunctionCallExpr(FunctionSet.BITMAP_UNION_COUNT, newParams);
- bitmapExpr.analyzeNoThrow(analyzer);
- return bitmapExpr;
- } else {
- FunctionCallExpr hllExpr = new FunctionCallExpr("hll_union_agg", newParams);
- hllExpr.analyzeNoThrow(analyzer);
- return hllExpr;
- }
- }
- return expr;
- }
/**
* Analyze aggregation-relevant components of the select block (Group By clause,
* select list, Order By clause),
@@ -853,6 +822,7 @@ public class SelectStmt extends QueryStmt {
* TODO: the a.key should be replaced by a.k1 instead of unknown column 'key' in 'a'
*/
havingClauseAfterAnaylzed = havingClause.substitute(aliasSMap, analyzer, false);
+ havingClauseAfterAnaylzed = rewriteCountDistinctForBitmapOrHLL(havingClauseAfterAnaylzed, analyzer);
havingClauseAfterAnaylzed.checkReturnsBool("HAVING clause", true);
// can't contain analytic exprs
Expr analyticExpr = havingClauseAfterAnaylzed.findFirstOf(AnalyticExpr.class);
diff --git a/fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index a979ebd..1f38371 100644
--- a/fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -371,6 +371,14 @@ public class QueryPlanTest {
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "explain " + sql);
Assert.assertTrue(explainString.contains("hll_union_agg"));
+ sql = "select count(distinct id2) from test.bitmap_table having count(distinct id2) > 0";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "explain " + sql);
+ Assert.assertTrue(explainString.contains("bitmap_union_count"));
+
+ sql = "select count(distinct id2) from test.bitmap_table order by count(distinct id2) > 0";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, "explain " + sql);
+ Assert.assertTrue(explainString.contains("bitmap_union_count"));
+
ConnectContext.get().getSessionVariable().setRewriteCountDistinct(false);
sql = "select count(distinct id2) from test.bitmap_table";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org