You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/01 11:01:48 UTC
(doris) 19/26: [Fix](nereids)Disable getting partition related table and column when self join (#30650)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7f0d3d9dcbdf56832ec1d936587a5e3edeeea457
Author: seawinde <14...@users.noreply.github.com>
AuthorDate: Thu Feb 1 18:33:55 2024 +0800
[Fix](nereids)Disable getting partition related table and column when self join (#30650)
* add left anti join ut
* forbidden the self join partition column get
* [Fix](nereids) Disable getting partition related table and column when self join
* fix code style
---
.../exploration/mv/MaterializedViewUtils.java | 56 ++++++++-----
.../exploration/mv/MaterializedViewUtilsTest.java | 92 +++++++++++++++++++++-
2 files changed, 126 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index e68f9d833c9..e7b11b5fd17 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -43,6 +43,9 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -72,9 +75,6 @@ public class MaterializedViewUtils {
break;
}
}
- if (columnExpr == null) {
- return Optional.empty();
- }
if (!(columnExpr instanceof SlotReference)) {
return Optional.empty();
}
@@ -85,13 +85,14 @@ public class MaterializedViewUtils {
// check sql pattern
IncrementCheckerContext context = new IncrementCheckerContext(columnSlot);
materializedViewPlan.accept(MaterializedViewIncrementChecker.INSTANCE, context);
- if (context.getTableColumnList().isEmpty() || !context.isPctPossible()) {
+ if (context.getPartitionRelatedTableAndColumnList().isEmpty() || !context.isPctPossible()) {
return Optional.empty();
}
// TODO support to return only one related table info, support multi later
- return Optional.of(new RelatedTableInfo(new BaseTableInfo(context.getTableColumnList().get(0).key()),
+ Pair<TableIf, Column> tableIfColumnPair = context.getPartitionRelatedTableAndColumnList().get(0);
+ return Optional.of(new RelatedTableInfo(new BaseTableInfo(tableIfColumnPair.key()),
context.isPctPossible(),
- context.getTableColumnList().get(0).value().getName()));
+ tableIfColumnPair.value().getName()));
}
/**
@@ -226,19 +227,29 @@ public class MaterializedViewUtils {
@Override
public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerContext context) {
- if (!(relation instanceof LogicalCatalogRelation) || !context.getTableColumnList().isEmpty()) {
- return visit(relation, context);
+ if (!(relation instanceof LogicalCatalogRelation)) {
+ return null;
}
LogicalCatalogRelation logicalCatalogRelation = (LogicalCatalogRelation) relation;
TableIf table = logicalCatalogRelation.getTable();
+ // if self join, can't infer partition column
+ if (!context.getTableIdAndRelationMapping().get(table.getId()).isEmpty()) {
+ context.setPctPossible(false);
+ return null;
+ }
+ // record tableId and relation, to check the self join
+ context.addTableIdAndRelation(((LogicalCatalogRelation) relation).getTable().getId(), relation);
+ // TODO: 2024/1/31 support only one partition referenced column, support multi later
+ if (!context.getPartitionRelatedTableAndColumnList().isEmpty()) {
+ return null;
+ }
if (!(table instanceof MTMVRelatedTableIf)) {
- return visit(relation, context);
+ return null;
}
MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table;
PartitionType type = relatedTable.getPartitionType();
-
if (PartitionType.UNPARTITIONED.equals(type)) {
- return visit(relation, context);
+ return null;
}
Set<Column> partitionColumnSet = new HashSet<>(relatedTable.getPartitionColumns());
Column mvReferenceColumn = context.getMvPartitionColumn().getColumn().get();
@@ -254,7 +265,8 @@ public class MaterializedViewUtils {
IncrementCheckerContext context) {
Set<Expression> groupByExprSet = new HashSet<>(aggregate.getGroupByExpressions());
if (groupByExprSet.isEmpty()) {
- return visit(aggregate, context);
+ context.setPctPossible(false);
+ return null;
}
Set<Column> originalGroupbyExprSet = new HashSet<>();
groupByExprSet.forEach(groupExpr -> {
@@ -264,6 +276,7 @@ public class MaterializedViewUtils {
});
if (!originalGroupbyExprSet.contains(context.getMvPartitionColumn().getColumn().get())) {
context.setPctPossible(false);
+ return null;
}
return visit(aggregate, context);
}
@@ -317,8 +330,9 @@ public class MaterializedViewUtils {
private static final class IncrementCheckerContext {
private final SlotReference mvPartitionColumn;
private boolean pctPossible = true;
- private final List<Pair<TableIf, Column>> tableColumnList = new ArrayList<>();
- private boolean joinNullGenerateSide;
+ private final List<Pair<TableIf, Column>> partitionRelatedTableAndColumnList = new ArrayList<>();
+ // This record the table id and relation mapping, because a table maybe used repeatedly.
+ private final Multimap<Long, LogicalRelation> tableIdAndRelationMapping = HashMultimap.create();
public IncrementCheckerContext(SlotReference mvPartitionColumn) {
this.mvPartitionColumn = mvPartitionColumn;
@@ -337,19 +351,19 @@ public class MaterializedViewUtils {
}
public void addTableColumn(TableIf relatedTable, Column partitionColumn) {
- tableColumnList.add(Pair.of(relatedTable, partitionColumn));
+ partitionRelatedTableAndColumnList.add(Pair.of(relatedTable, partitionColumn));
}
- public List<Pair<TableIf, Column>> getTableColumnList() {
- return tableColumnList;
+ public List<Pair<TableIf, Column>> getPartitionRelatedTableAndColumnList() {
+ return partitionRelatedTableAndColumnList;
}
- public boolean isJoinNullGenerateSide() {
- return joinNullGenerateSide;
+ public Multimap<Long, LogicalRelation> getTableIdAndRelationMapping() {
+ return tableIdAndRelationMapping;
}
- public void setJoinNullGenerateSide(boolean joinNullGenerateSide) {
- this.joinNullGenerateSide = joinNullGenerateSide;
+ public void addTableIdAndRelation(Long tableId, LogicalRelation relation) {
+ tableIdAndRelationMapping.put(tableId, relation);
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
index 8bf8ea14ea5..4204bbe0221 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
@@ -65,6 +65,37 @@ public class MaterializedViewUtilsTest extends TestWithFeService {
+ "PROPERTIES (\n"
+ " \"replication_num\" = \"1\"\n"
+ ")");
+
+ createTable("CREATE TABLE `lineitem_list_partition` (\n"
+ + " `l_orderkey` BIGINT not NULL,\n"
+ + " `l_linenumber` INT NULL,\n"
+ + " `l_partkey` INT NULL,\n"
+ + " `l_suppkey` INT NULL,\n"
+ + " `l_quantity` DECIMAL(15, 2) NULL,\n"
+ + " `l_extendedprice` DECIMAL(15, 2) NULL,\n"
+ + " `l_discount` DECIMAL(15, 2) NULL,\n"
+ + " `l_tax` DECIMAL(15, 2) NULL,\n"
+ + " `l_returnflag` VARCHAR(1) NULL,\n"
+ + " `l_linestatus` VARCHAR(1) NULL,\n"
+ + " `l_commitdate` DATE NULL,\n"
+ + " `l_receiptdate` DATE NULL,\n"
+ + " `l_shipinstruct` VARCHAR(25) NULL,\n"
+ + " `l_shipmode` VARCHAR(10) NULL,\n"
+ + " `l_comment` VARCHAR(44) NULL,\n"
+ + " `l_shipdate` DATE NULL\n"
+ + " ) ENGINE=OLAP\n"
+ + " DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )\n"
+ + " COMMENT 'OLAP'\n"
+ + " PARTITION BY list(l_orderkey) (\n"
+ + " PARTITION p1 VALUES in ('1'),\n"
+ + " PARTITION p2 VALUES in ('2'),\n"
+ + " PARTITION p3 VALUES in ('3')\n"
+ + " )\n"
+ + " DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 3\n"
+ + " PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + " )");
+
createTable("CREATE TABLE IF NOT EXISTS orders (\n"
+ " O_ORDERKEY INTEGER NOT NULL,\n"
+ " O_CUSTKEY INTEGER NOT NULL,\n"
@@ -82,6 +113,30 @@ public class MaterializedViewUtilsTest extends TestWithFeService {
+ "PROPERTIES (\n"
+ " \"replication_num\" = \"1\"\n"
+ ")");
+
+ createTable("CREATE TABLE `orders_list_partition` (\n"
+ + " `o_orderkey` BIGINT not NULL,\n"
+ + " `o_custkey` INT NULL,\n"
+ + " `o_orderstatus` VARCHAR(1) NULL,\n"
+ + " `o_totalprice` DECIMAL(15, 2) NULL,\n"
+ + " `o_orderpriority` VARCHAR(15) NULL,\n"
+ + " `o_clerk` VARCHAR(15) NULL,\n"
+ + " `o_shippriority` INT NULL,\n"
+ + " `o_comment` VARCHAR(79) NULL,\n"
+ + " `o_orderdate` DATE NULL\n"
+ + " ) ENGINE=OLAP\n"
+ + " DUPLICATE KEY(`o_orderkey`, `o_custkey`)\n"
+ + " COMMENT 'OLAP'\n"
+ + " PARTITION BY list(o_orderkey) (\n"
+ + " PARTITION p1 VALUES in ('1'),\n"
+ + " PARTITION p2 VALUES in ('2'),\n"
+ + " PARTITION p3 VALUES in ('3'),\n"
+ + " PARTITION p4 VALUES in ('4')\n"
+ + " )\n"
+ + " DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 3\n"
+ + " PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + " )");
createTable("CREATE TABLE IF NOT EXISTS partsupp (\n"
+ " PS_PARTKEY INTEGER NOT NULL,\n"
+ " PS_SUPPKEY INTEGER NOT NULL,\n"
@@ -141,7 +196,7 @@ public class MaterializedViewUtilsTest extends TestWithFeService {
+ " DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )\n"
+ " COMMENT 'OLAP'\n"
+ " AUTO PARTITION BY range date_trunc(`l_shipdate`, 'day') ()\n"
- + " DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96\n"
+ + " DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 3\n"
+ " PROPERTIES (\n"
+ " \"replication_num\" = \"1\"\n"
+ " );\n"
@@ -276,6 +331,41 @@ public class MaterializedViewUtilsTest extends TestWithFeService {
});
}
+ @Test
+ public void getRelatedTableInfoLeftAntiJoinTest() {
+ PlanChecker.from(connectContext)
+ .checkExplain(" select l_shipdate, l_orderkey, count(l_shipdate), count(l_orderkey) \n"
+ + " from lineitem_list_partition\n"
+ + " left anti join orders_list_partition\n"
+ + " on l_shipdate = o_orderdate\n"
+ + " group by l_shipdate, l_orderkey",
+ nereidsPlanner -> {
+ Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan();
+ Optional<RelatedTableInfo> relatedTableInfo =
+ MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan);
+ checkRelatedTableInfo(relatedTableInfo,
+ "lineitem_list_partition",
+ "l_orderkey",
+ true);
+ });
+ }
+
+ @Test
+ public void getRelatedTableInfoSelfJoinTest() {
+ PlanChecker.from(connectContext)
+ .checkExplain(" select t1.l_shipdate, t1.l_orderkey, t1.l_partkey, t1.l_suppkey, 1\n"
+ + " from lineitem_list_partition t1\n"
+ + " join lineitem_list_partition t2\n"
+ + " on t1.l_shipdate = t2.l_shipdate\n"
+ + " group by t1.l_shipdate, t1.l_orderkey, t1.l_partkey, t1.l_suppkey",
+ nereidsPlanner -> {
+ Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan();
+ Optional<RelatedTableInfo> relatedTableInfo =
+ MaterializedViewUtils.getRelatedTableInfo("l_orderkey", rewrittenPlan);
+ Assertions.assertFalse(relatedTableInfo.isPresent());
+ });
+ }
+
@Test
public void getRelatedTableInfoUseRightTest() {
PlanChecker.from(connectContext)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org