You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2024/04/26 11:31:17 UTC
(doris) branch master updated: [improvement](mtmv) Optimize the nested materialized view performance (#34163)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 935f613ce83 [improvement](mtmv) Optimize the nested materialized view performance (#34163)
935f613ce83 is described below
commit 935f613ce831193467f8d8192d57f2f01cf90744
Author: seawinde <14...@users.noreply.github.com>
AuthorDate: Fri Apr 26 19:31:09 2024 +0800
[improvement](mtmv) Optimize the nested materialized view performance (#34163)
Record increase refersh version more accurately.
The refreshVersion in the memo will increase when mv rewrite successfully.
In query rewrite, if refresh version is different from the current struct info map in group,
will refresh the group struct info or not.
---
.../src/main/java/org/apache/doris/nereids/memo/Memo.java | 4 +++-
.../java/org/apache/doris/nereids/memo/StructInfoMap.java | 14 ++++++++------
.../rules/exploration/mv/MaterializedViewUtils.java | 3 +--
.../org/apache/doris/nereids/memo/StructInfoMapTest.java | 14 +++++++-------
4 files changed, 19 insertions(+), 16 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index 8793cb5be51..1e6b285187d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -416,7 +416,9 @@ public class Memo {
throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties");
}
// TODO Support sync materialized view in the future
- if (plan instanceof CatalogRelation && ((CatalogRelation) plan).getTable() instanceof MTMV) {
+ if (plan instanceof LogicalPlan && plan instanceof CatalogRelation
+ && ((CatalogRelation) plan).getTable() instanceof MTMV
+ && !plan.getGroupExpression().isPresent()) {
refreshVersion.incrementAndGet();
}
Optional<GroupExpression> groupExpr = plan.getGroupExpression();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
index efa2bef1792..ae07c2043ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
@@ -59,7 +59,7 @@ public class StructInfoMap {
return structInfo;
}
if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) {
- refresh(group, memo.getRefreshVersion(), foldTableMap);
+ refresh(group, memo.getRefreshVersion());
group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion());
}
if (groupExpressionMap.containsKey(tableMap)) {
@@ -118,7 +118,10 @@ public class StructInfoMap {
* @param group the root group
*
*/
- public void refresh(Group group, long refreshVersion, BitSet targetBitSet) {
+ public void refresh(Group group, long memoVersion) {
+ if (memoVersion == group.getstructInfoMap().refreshVersion) {
+ return;
+ }
Set<Integer> refreshedGroup = new HashSet<>();
for (GroupExpression groupExpression : group.getLogicalExpressions()) {
List<Set<BitSet>> childrenTableMap = new LinkedList<>();
@@ -129,10 +132,9 @@ public class StructInfoMap {
}
for (Group child : groupExpression.children()) {
StructInfoMap childStructInfoMap = child.getstructInfoMap();
- if (!refreshedGroup.contains(child.getGroupId().asInt())
- && refreshVersion != childStructInfoMap.getRefreshVersion()) {
- childStructInfoMap.refresh(child, refreshVersion, targetBitSet);
- childStructInfoMap.setRefreshVersion(refreshVersion);
+ if (!refreshedGroup.contains(child.getGroupId().asInt())) {
+ childStructInfoMap.refresh(child, memoVersion);
+ childStructInfoMap.setRefreshVersion(memoVersion);
}
refreshedGroup.add(child.getGroupId().asInt());
childrenTableMap.add(child.getstructInfoMap().getTableMaps());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 5f7dc419eaf..73029b4ec14 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -150,8 +150,7 @@ public class MaterializedViewUtils {
StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
if (cascadesContext.getMemo().getRefreshVersion() != structInfoMap.getRefreshVersion()
|| structInfoMap.getTableMaps().isEmpty()) {
- structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion(),
- materializedViewTableSet);
+ structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion());
structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
}
Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 9192f86cf3b..217cbf2f1a2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -50,7 +50,7 @@ class StructInfoMapTest extends SqlTestBase {
Group root = c1.getMemo().getRoot();
Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertTrue(tableMaps.isEmpty());
- root.getstructInfoMap().refresh(root, 1, new BitSet());
+ root.getstructInfoMap().refresh(root, 1);
Assertions.assertEquals(1, tableMaps.size());
new MockUp<MTMVRelationManager>() {
@Mock
@@ -76,7 +76,7 @@ class StructInfoMapTest extends SqlTestBase {
.optimize()
.printlnBestPlanTree();
root = c1.getMemo().getRoot();
- root.getstructInfoMap().refresh(root, 1, new BitSet());
+ root.getstructInfoMap().refresh(root, 1);
tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertEquals(2, tableMaps.size());
dropMvByNereids("drop materialized view mv1");
@@ -97,8 +97,8 @@ class StructInfoMapTest extends SqlTestBase {
Group root = c1.getMemo().getRoot();
Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertTrue(tableMaps.isEmpty());
- root.getstructInfoMap().refresh(root, 1, new BitSet());
- root.getstructInfoMap().refresh(root, 1, new BitSet());
+ root.getstructInfoMap().refresh(root, 1);
+ root.getstructInfoMap().refresh(root, 1);
Assertions.assertEquals(1, tableMaps.size());
new MockUp<MTMVRelationManager>() {
@Mock
@@ -124,8 +124,8 @@ class StructInfoMapTest extends SqlTestBase {
.optimize()
.printlnBestPlanTree();
root = c1.getMemo().getRoot();
- root.getstructInfoMap().refresh(root, 1, new BitSet());
- root.getstructInfoMap().refresh(root, 1, new BitSet());
+ root.getstructInfoMap().refresh(root, 1);
+ root.getstructInfoMap().refresh(root, 1);
tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertEquals(2, tableMaps.size());
dropMvByNereids("drop materialized view mv1");
@@ -162,7 +162,7 @@ class StructInfoMapTest extends SqlTestBase {
.rewrite()
.optimize();
Group root = c1.getMemo().getRoot();
- root.getstructInfoMap().refresh(root, 1, new BitSet());
+ root.getstructInfoMap().refresh(root, 1);
StructInfoMap structInfoMap = root.getstructInfoMap();
Assertions.assertEquals(2, structInfoMap.getTableMaps().size());
BitSet mvMap = structInfoMap.getTableMaps().stream()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org