You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ya...@apache.org on 2022/12/22 11:25:17 UTC
[doris] branch master updated: [fix](Inbitmap) fix in bitmap result error when left expr is constant (#15271)
This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1fdd4172bd [fix](Inbitmap) fix in bitmap result error when left expr is constant (#15271)
1fdd4172bd is described below
commit 1fdd4172bddf3ccd687c3331cc0a1f18de260169
Author: luozenglin <37...@users.noreply.github.com>
AuthorDate: Thu Dec 22 19:25:09 2022 +0800
[fix](Inbitmap) fix in bitmap result error when left expr is constant (#15271)
* [fix](Inbitmap) fix in bitmap result error when left expr is constant
1. When left expr of the in predicate is a constant, instead of generating a bitmap filter, rewrite sql to use `bitmap_contains`.
For example,"select k1, k2 from (select 2 k1, 11 k2) t where k1 in (select bitmap_col from bitmap_tbl)"
=> "select k1, k2 from (select 2 k1, 11 k2) t left semi join bitmap_tbl b on bitmap_contains(b.bitmap_col, t.k1)"
* add regression test
---
.../org/apache/doris/analysis/StmtRewriter.java | 45 +++++++++++++++++-----
.../data/query_p0/join/test_bitmap_filter.out | 10 +++++
.../suites/query_p0/join/test_bitmap_filter.groovy | 8 ++++
3 files changed, 53 insertions(+), 10 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
index f837bf6af5..a93c21692c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java
@@ -50,6 +50,8 @@ import java.util.List;
public class StmtRewriter {
private static final Logger LOG = LoggerFactory.getLogger(StmtRewriter.class);
+ private static final String BITMAP_CONTAINS = "bitmap_contains";
+
/**
* Rewrite the statement of an analysis result. The unanalyzed rewritten
* statement is returned.
@@ -799,8 +801,12 @@ public class StmtRewriter {
if (!hasEqJoinPred && !inlineView.isCorrelated()) {
// Join with InPredicate is actually an equal join, so we choose HashJoin.
if (expr instanceof ExistsPredicate) {
- joinOp = ((ExistsPredicate) expr).isNotExists() ? JoinOperator.LEFT_ANTI_JOIN
- : JoinOperator.LEFT_SEMI_JOIN;
+ joinOp = ((ExistsPredicate) expr).isNotExists() ? JoinOperator.LEFT_ANTI_JOIN :
+ JoinOperator.LEFT_SEMI_JOIN;
+ } else if (expr instanceof InPredicate && joinConjunct instanceof FunctionCallExpr
+ && (((FunctionCallExpr) joinConjunct).getFnName().getFunction()
+ .equalsIgnoreCase(BITMAP_CONTAINS))) {
+ joinOp = ((InPredicate) expr).isNotIn() ? JoinOperator.LEFT_ANTI_JOIN : JoinOperator.LEFT_SEMI_JOIN;
} else {
joinOp = JoinOperator.CROSS_JOIN;
// We can equal the aggregate subquery using a cross join. All conjuncts
@@ -1166,6 +1172,32 @@ public class StmtRewriter {
}
}
+ // If left expr of in predicate is a constant and a bitmap filter cannot be used, then the normal nested loop join
+ // process is used, with the join Conjunct being `bitmap_contains`,
+ // e.g. 'select k1, k2 from (select 2 k1, 11 k2) t where k1 in (select bitmap_col from bitmap_tbl)'.
+ private static Expr createInBitmapConjunct(Expr exprWithSubquery, SlotRef bitmapSlotRef, Analyzer analyzer,
+ boolean isCorrelated) throws AnalysisException {
+ if (isCorrelated) {
+ throw new AnalysisException("In bitmap does not support correlated subquery: " + exprWithSubquery.toSql());
+ }
+
+ boolean useBitmapFilter = false;
+ List<SlotRef> slotRefs = Lists.newArrayList();
+ exprWithSubquery.getChild(0).collect(SlotRef.class, slotRefs);
+ for (SlotRef slotRef : slotRefs) {
+ List<Expr> sourceExprs = slotRef.getDesc().getSourceExprs();
+ if (sourceExprs.isEmpty() || sourceExprs.stream().anyMatch(expr -> !expr.isConstant())) {
+ useBitmapFilter = true;
+ }
+ }
+
+ Expr pred = useBitmapFilter ? new BitmapFilterPredicate(exprWithSubquery.getChild(0), bitmapSlotRef,
+ ((InPredicate) exprWithSubquery).isNotIn()) : new FunctionCallExpr(new FunctionName(BITMAP_CONTAINS),
+ Lists.newArrayList(bitmapSlotRef, exprWithSubquery.getChild(0)));
+ pred.analyze(analyzer);
+ return pred;
+ }
+
/**
* Converts an expr containing a subquery into an analyzed conjunct to be
* used in a join. The conversion is performed in place by replacing the
@@ -1189,14 +1221,7 @@ public class StmtRewriter {
Expr subquerySubstitute = slotRef;
if (exprWithSubquery instanceof InPredicate) {
if (slotRef.getType().isBitmapType()) {
- if (isCorrelated) {
- throw new AnalysisException(
- "In bitmap does not support correlated subquery: " + exprWithSubquery.toSql());
- }
- Expr pred = new BitmapFilterPredicate(exprWithSubquery.getChild(0), slotRef,
- ((InPredicate) exprWithSubquery).isNotIn());
- pred.analyze(analyzer);
- return pred;
+ return createInBitmapConjunct(exprWithSubquery, slotRef, analyzer, isCorrelated);
}
BinaryPredicate pred = new BinaryPredicate(BinaryPredicate.Operator.EQ,
exprWithSubquery.getChild(0), slotRef);
diff --git a/regression-test/data/query_p0/join/test_bitmap_filter.out b/regression-test/data/query_p0/join/test_bitmap_filter.out
index 24d7773720..83bcb3aed9 100644
--- a/regression-test/data/query_p0/join/test_bitmap_filter.out
+++ b/regression-test/data/query_p0/join/test_bitmap_filter.out
@@ -56,3 +56,13 @@
1991 2
32767 1
+-- !sql7 --
+
+-- !sql8 --
+11 11
+
+-- !sql9 --
+2 11
+
+-- !sql10 --
+
diff --git a/regression-test/suites/query_p0/join/test_bitmap_filter.groovy b/regression-test/suites/query_p0/join/test_bitmap_filter.groovy
index eaa55bc0eb..92b99052a3 100644
--- a/regression-test/suites/query_p0/join/test_bitmap_filter.groovy
+++ b/regression-test/suites/query_p0/join/test_bitmap_filter.groovy
@@ -50,6 +50,14 @@ suite("test_bitmap_filter", "query_p0") {
qt_sql6 "select k2, count(k2) from ${tbl1} where k1 in (select k2 from ${tbl2}) group by k2 order by k2;"
+ qt_sql7 "select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from ${tbl2}) order by 1, 2;"
+
+ qt_sql8 "select k1, k2 from (select 11 k1, 11 k2) t where k1 in (select k2 from ${tbl2}) order by 1, 2;"
+
+ qt_sql9 "select k1, k2 from (select 2 k1, 11 k2) t where k1 not in (select k2 from ${tbl2}) order by 1, 2;"
+
+ qt_sql10 "select k1, k2 from (select 1 k1, 11 k2) t where k1 not in (select k2 from ${tbl2}) order by 1, 2;"
+
test {
sql "select k1, k2 from ${tbl1} b1 where k1 in (select k2 from ${tbl2} b2 where b1.k2 = b2.k1) order by k1;"
exception "In bitmap does not support correlated subquery"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org