You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/03/12 10:50:45 UTC
(doris) 02/07: [feat](Nereids): use table map to construct struct info (#32058)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2da57526a3098ae009288ae478407beccee31bc0
Author: 谢健 <ji...@gmail.com>
AuthorDate: Tue Mar 12 15:52:36 2024 +0800
[feat](Nereids): use table map to construct struct info (#32058)
---
.../apache/doris/nereids/memo/StructInfoMap.java | 46 ++++++++++++++++-----
.../doris/nereids/memo/StructInfoMapTest.java | 48 ++++++++++++++++++++++
.../apache/doris/utframe/TestWithFeService.java | 25 +++++++++++
3 files changed, 108 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
index f7e09fd44d9..c90cc35c85d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.memo;
+import org.apache.doris.common.Pair;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
@@ -37,11 +38,12 @@ import javax.annotation.Nullable;
* Representation for group in cascades optimizer.
*/
public class StructInfoMap {
- private final Map<BitSet, GroupExpression> groupExpressionMap = new HashMap<>();
+ private final Map<BitSet, Pair<GroupExpression, List<BitSet>>> groupExpressionMap = new HashMap<>();
private final Map<BitSet, StructInfo> infoMap = new HashMap<>();
/**
* get struct info according to table map
+ *
* @param mvTableMap the original table map
* @param foldTableMap the fold table map
* @param group the group that the mv matched
@@ -54,7 +56,10 @@ public class StructInfoMap {
refresh(group);
}
if (groupExpressionMap.containsKey(mvTableMap)) {
- StructInfo structInfo = constructStructInfo(groupExpressionMap.get(mvTableMap));
+ Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair = getGroupExpressionWithChildren(
+ mvTableMap);
+ StructInfo structInfo = constructStructInfo(groupExpressionBitSetPair.first,
+ groupExpressionBitSetPair.second, mvTableMap);
infoMap.put(mvTableMap, structInfo);
}
}
@@ -66,20 +71,39 @@ public class StructInfoMap {
return groupExpressionMap.keySet();
}
- private StructInfo constructStructInfo(GroupExpression groupExpression) {
- throw new RuntimeException("has not been implemented for" + groupExpression);
+ public Pair<GroupExpression, List<BitSet>> getGroupExpressionWithChildren(BitSet tableMap) {
+ return groupExpressionMap.get(tableMap);
+ }
+
+ private StructInfo constructStructInfo(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) {
+ Plan plan = constructPlan(groupExpression, children, tableMap);
+ return StructInfo.of(plan).get(0);
+ }
+
+ private Plan constructPlan(GroupExpression groupExpression, List<BitSet> children, BitSet tableMap) {
+ List<Plan> childrenPlan = new ArrayList<>();
+ for (int i = 0; i < children.size(); i++) {
+ StructInfoMap structInfoMap = groupExpression.child(i).getstructInfoMap();
+ BitSet childMap = children.get(i);
+ Pair<GroupExpression, List<BitSet>> groupExpressionBitSetPair
+ = structInfoMap.getGroupExpressionWithChildren(childMap);
+ childrenPlan.add(
+ constructPlan(groupExpressionBitSetPair.first, groupExpressionBitSetPair.second, childMap));
+ }
+ return groupExpression.getPlan().withChildren(childrenPlan);
}
/**
* refresh group expression map
+ *
* @param group the root group
*/
public void refresh(Group group) {
- List<Set<BitSet>> childrenTableMap = new ArrayList<>();
Set<Group> refreshedGroup = new HashSet<>();
for (GroupExpression groupExpression : group.getLogicalExpressions()) {
+ List<Set<BitSet>> childrenTableMap = new ArrayList<>();
if (groupExpression.children().isEmpty()) {
- groupExpressionMap.put(constructLeaf(groupExpression), groupExpression);
+ groupExpressionMap.put(constructLeaf(groupExpression), Pair.of(groupExpression, new ArrayList<>()));
continue;
}
for (Group child : groupExpression.children()) {
@@ -90,9 +114,9 @@ public class StructInfoMap {
refreshedGroup.add(child);
childrenTableMap.add(child.getstructInfoMap().getTableMaps());
}
- Set<BitSet> bitSets = cartesianProduct(childrenTableMap);
- for (BitSet bitSet : bitSets) {
- groupExpressionMap.put(bitSet, groupExpression);
+ Set<Pair<BitSet, List<BitSet>>> bitSetWithChildren = cartesianProduct(childrenTableMap);
+ for (Pair<BitSet, List<BitSet>> bitSetWithChild : bitSetWithChildren) {
+ groupExpressionMap.put(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second));
}
}
}
@@ -108,7 +132,7 @@ public class StructInfoMap {
return tableMap;
}
- private Set<BitSet> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
+ private Set<Pair<BitSet, List<BitSet>>> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
return Sets.cartesianProduct(childrenTableMap)
.stream()
.map(bitSetList -> {
@@ -116,7 +140,7 @@ public class StructInfoMap {
for (BitSet b : bitSetList) {
bitSet.or(b);
}
- return bitSet;
+ return Pair.of(bitSet, bitSetList);
})
.collect(Collectors.toSet());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 0699f04b92f..08db05b5065 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.memo;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.mtmv.MTMVRelationManager;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.qe.ConnectContext;
@@ -31,6 +32,7 @@ import org.junit.jupiter.api.Test;
import java.util.BitSet;
import java.util.Set;
+import java.util.stream.Collectors;
class StructInfoMapTest extends SqlTestBase {
@Test
@@ -77,5 +79,51 @@ class StructInfoMapTest extends SqlTestBase {
root.getstructInfoMap().refresh(root);
tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertEquals(2, tableMaps.size());
+ dropMvByNereids("drop materialized view mv1");
+ }
+
+ @Test
+ void testTableChild() throws Exception {
+ CascadesContext c1 = createCascadesContext(
+ "select T1.id from T1 inner join T2 "
+ + "on T1.id = T2.id "
+ + "inner join T3 on T1.id = T3.id",
+ connectContext
+ );
+ new MockUp<MTMVRelationManager>() {
+ @Mock
+ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
+ return true;
+ }
+ };
+ connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
+ createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select T1.id from T1 inner join T2 "
+ + "on T1.id = T2.id;");
+ c1 = createCascadesContext(
+ "select T1.id from T1 inner join T2 "
+ + "on T1.id = T2.id "
+ + "inner join T3 on T1.id = T3.id",
+ connectContext
+ );
+ PlanChecker.from(c1)
+ .analyze()
+ .rewrite()
+ .optimize();
+ Group root = c1.getMemo().getRoot();
+ root.getstructInfoMap().refresh(root);
+ StructInfoMap structInfoMap = root.getstructInfoMap();
+ Assertions.assertEquals(2, structInfoMap.getTableMaps().size());
+ BitSet mvMap = structInfoMap.getTableMaps().stream()
+ .filter(b -> b.cardinality() == 2)
+ .collect(Collectors.toList()).get(0);
+ StructInfo structInfo = structInfoMap.getStructInfo(mvMap, mvMap, root);
+ System.out.println(structInfo.getOriginalPlan().treeString());
+ BitSet bitSet = new BitSet();
+ structInfo.getRelations().forEach(r -> bitSet.set((int) r.getTable().getId()));
+ Assertions.assertEquals(bitSet, mvMap);
+ dropMvByNereids("drop materialized view mv1");
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index 7de8a9a830b..c43ab447885 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -69,6 +69,7 @@ import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.persist.CreateTableInfo;
@@ -823,6 +824,30 @@ public abstract class TestWithFeService {
}
+ protected void dropMvByNereids(String sql) throws Exception {
+ new MockUp<EditLog>() {
+ @Mock
+ public void logCreateTable(CreateTableInfo info) {
+ System.out.println("skip log create table...");
+ }
+
+ @Mock
+ public void logCreateJob(AbstractJob job) {
+ System.out.println("skip log create job...");
+ }
+ };
+ NereidsParser nereidsParser = new NereidsParser();
+ LogicalPlan parsed = nereidsParser.parseSingle(sql);
+ StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
+ if (parsed instanceof DropMTMVCommand) {
+ ((DropMTMVCommand) parsed).run(connectContext, stmtExecutor);
+ }
+ checkAlterJob();
+ // waiting table state to normal
+ Thread.sleep(1000);
+
+ }
+
private void updateReplicaPathHash() {
com.google.common.collect.Table<Long, Long, Replica> replicaMetaTable = Env.getCurrentInvertedIndex()
.getReplicaMetaTable();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org