You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2024/07/15 08:30:42 UTC
(kylin) 16/30: KYLIN-5865 Supports OlapFilterJoinRule.FILTER_ON_JOIN
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f2d5f586ad87196621d3a93d5133f3595cfc5b4b
Author: Hao Chen <ha...@qq.com>
AuthorDate: Sat Nov 4 16:10:54 2023 +0800
KYLIN-5865 Supports OlapFilterJoinRule.FILTER_ON_JOIN
Co-authored-by: hao.chen <ha...@kyligence.io>
---
pom.xml | 2 +-
.../kylin/query/engine/SumExprPlannerTest.java | 7 ++-
.../kylin/query/engine/SumExprPlannerTest.xml | 11 +++--
.../org/apache/kylin/query/engine/QueryExec.java | 16 +++----
.../kylin/query/optrule/OlapFilterJoinRule.java | 53 ++++++++++++++++++++++
5 files changed, 74 insertions(+), 15 deletions(-)
diff --git a/pom.xml b/pom.xml
index 8c1883868c..091ee90583 100644
--- a/pom.xml
+++ b/pom.xml
@@ -138,7 +138,7 @@
<scala-retry>0.3.0</scala-retry>
<!-- Calcite Version -->
- <calcite.version>1.116.0-kylin-4.x-r037</calcite.version>
+ <calcite.version>1.116.0-kylin-4.x-r038</calcite.version>
<avatica.version>4.x_1.10-r01</avatica.version>
<!-- Hadoop Common deps, keep compatible with hadoop2.version -->
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
index fb73616ba3..7472e9368a 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/SumExprPlannerTest.java
@@ -21,16 +21,19 @@ package org.apache.kylin.query.engine;
import java.io.IOException;
import java.util.List;
+import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.test.DiffRepository;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.query.optrule.OlapAggFilterTransposeRule;
import org.apache.kylin.query.optrule.OlapAggJoinTransposeRule;
import org.apache.kylin.query.optrule.OlapAggProjectMergeRule;
import org.apache.kylin.query.optrule.OlapAggProjectTransposeRule;
import org.apache.kylin.query.optrule.OlapAggregateRule;
import org.apache.kylin.query.optrule.OlapCountDistinctJoinRule;
+import org.apache.kylin.query.optrule.OlapFilterJoinRule;
import org.apache.kylin.query.optrule.OlapJoinRule;
import org.apache.kylin.query.optrule.OlapProjectRule;
import org.apache.kylin.query.optrule.OlapSumCastTransposeRule;
@@ -203,7 +206,9 @@ public class SumExprPlannerTest extends CalciteRuleTestBase {
+ " ) a\n" //
+ "where LO_COMMITDATE = ( select max(LO_COMMITDATE) from ssb.P_LINEORDER )";
openSumCaseWhen();
- checkSQL("ssb", sql, null, null);
+ List<RelOptRule> rules = Lists.newArrayList(HepUtils.SumExprRules);
+ rules.add(OlapFilterJoinRule.FILTER_ON_JOIN);
+ super.checkSQLPostOptimize("ssb", sql, null, null, rules);
}
@Test
diff --git a/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml b/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
index b112b5ab17..95a3e8ab94 100644
--- a/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
+++ b/src/kylin-it/src/test/resources/org/apache/kylin/query/engine/SumExprPlannerTest.xml
@@ -1446,12 +1446,13 @@ OlapToEnumerableConverter
OlapToEnumerableConverter
OlapAggregateRel(group-set=[[]], groups=[null], EXPR$0=[SUM($0)], ctx=[])
OlapProjectRel($f0=[CASE(=($0, 20230501), $1, null)], ctx=[])
- OlapJoinRel(condition=[=($0, $3)], joinType=[inner], ctx=[])
- OlapProjectRel(LO_COMMITDATE=[$15], LO_DISCOUNT=[$11], LO_ORDERDATE=[$5], ctx=[])
- OlapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
- OlapAggregateRel(group-set=[[]], groups=[null], EXPR$0=[MAX($0)], ctx=[])
- OlapProjectRel(LO_COMMITDATE=[$15], ctx=[])
+ OlapFilterRel(condition=[=($0, $3)], ctx=[])
+ OlapJoinRel(condition=[true], joinType=[left], ctx=[])
+ OlapProjectRel(LO_COMMITDATE=[$15], LO_DISCOUNT=[$11], LO_ORDERDATE=[$5], ctx=[])
OlapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
+ OlapAggregateRel(group-set=[[]], groups=[null], EXPR$0=[MAX($0)], ctx=[])
+ OlapProjectRel(LO_COMMITDATE=[$15], ctx=[])
+ OlapTableScan(table=[[SSB, P_LINEORDER]], ctx=[], fields=[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]])
]]>
</Resource>
<Resource name="planAfter">
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
index 775069338e..122a23b3db 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryExec.java
@@ -68,6 +68,7 @@ import org.apache.kylin.query.engine.exec.SparderPlanExec;
import org.apache.kylin.query.engine.meta.SimpleDataContext;
import org.apache.kylin.query.engine.view.ViewAnalyzer;
import org.apache.kylin.query.mask.QueryResultMasks;
+import org.apache.kylin.query.optrule.OlapFilterJoinRule;
import org.apache.kylin.query.optrule.SumConstantConvertRule;
import org.apache.kylin.query.relnode.ContextUtil;
import org.apache.kylin.query.relnode.OlapAggregateRel;
@@ -291,15 +292,14 @@ public class QueryExec {
postOptRules.addAll(HepUtils.FilterReductionRules);
}
- if (!postOptRules.isEmpty()) {
- RelNode transformed = HepUtils.runRuleCollection(node, postOptRules, false);
- if (transformed != node && allowAlternativeQueryPlan) {
- return Lists.newArrayList(transformed, node);
- } else {
- return Lists.newArrayList(transformed);
- }
+ postOptRules.add(OlapFilterJoinRule.FILTER_ON_JOIN);
+
+ RelNode transformed = HepUtils.runRuleCollection(node, postOptRules, false);
+ if (transformed != node && allowAlternativeQueryPlan) {
+ return Lists.newArrayList(transformed, node);
+ } else {
+ return Lists.newArrayList(transformed);
}
- return Lists.newArrayList(node);
}
@VisibleForTesting
diff --git a/src/query/src/main/java/org/apache/kylin/query/optrule/OlapFilterJoinRule.java b/src/query/src/main/java/org/apache/kylin/query/optrule/OlapFilterJoinRule.java
index b04d65b991..26ffe41ddb 100644
--- a/src/query/src/main/java/org/apache/kylin/query/optrule/OlapFilterJoinRule.java
+++ b/src/query/src/main/java/org/apache/kylin/query/optrule/OlapFilterJoinRule.java
@@ -21,6 +21,7 @@ package org.apache.kylin.query.optrule;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
import org.apache.calcite.plan.RelOptRule;
@@ -35,9 +36,12 @@ import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.rules.FilterJoinRule;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexPermuteInputsShuttle;
import org.apache.calcite.rex.RexUtil;
@@ -48,6 +52,7 @@ import org.apache.calcite.util.mapping.Mappings;
import org.apache.kylin.guava30.shaded.common.collect.ImmutableList;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
+import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import org.apache.kylin.query.util.RexUtils;
/**
@@ -56,6 +61,54 @@ import org.apache.kylin.query.util.RexUtils;
*
*/
public class OlapFilterJoinRule extends RelOptRule {
+
+ public static final FilterJoinRule FILTER_ON_JOIN = new FilterJoinRule.FilterIntoJoinRule(true,
+ RelFactories.LOGICAL_BUILDER, FilterJoinRule.TRUE_PREDICATE) {
+
+ // https://olapio.atlassian.net/browse/AL-8813
+ @Override
+ protected boolean canPushIntoFromAbove(Filter filter) {
+ if (filter == null) {
+ return false;
+ }
+
+ if (isFilterContainsCC(filter)) {
+ return false;
+ }
+
+ RexNode condition = filter.getCondition();
+ if (condition.isA(SqlKind.AND)) {
+ RexCall call = (RexCall) condition;
+ return call.getOperands().stream().allMatch(this::isSimpleInputRefCondition);
+ }
+ return isSimpleInputRefCondition(condition);
+ }
+
+ private boolean isFilterContainsCC(Filter filter) {
+ RexNode condition = filter.getCondition();
+ if (condition == null) {
+ return false;
+ }
+
+ Set<RexInputRef> allInputRefs = RexUtils.getAllInputRefs(condition);
+ List<RelDataTypeField> fieldList = filter.getRowType().getFieldList();
+ return allInputRefs.stream().anyMatch(inputRef -> fieldList.get(inputRef.getIndex()).getName()
+ .startsWith(ComputedColumnUtil.CC_NAME_PREFIX));
+ }
+
+ // for example: table1.col1 = table2.col1 can be pushed into the below join relNode,
+ // however, table1.col1 + table1.col2 = table2.col1 can't be pushed.
+ private boolean isSimpleInputRefCondition(RexNode condition) {
+ if (condition.isA(SqlKind.EQUALS)) {
+ RexCall call = (RexCall) condition;
+ RexNode left = call.getOperands().get(0);
+ RexNode right = call.getOperands().get(1);
+ return left instanceof RexInputRef && right instanceof RexInputRef;
+ }
+ return false;
+ }
+ };
+
public static final OlapFilterJoinRule OLAP_FILTER_ON_JOIN_JOIN = new OlapFilterJoinRule(
operand(Filter.class,
operand(Join.class,