You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2025/04/08 07:18:32 UTC
(doris) branch branch-2.1 updated: [fix](mtmv) Fix materialized rewrite oom when the num of relation mapping is too large (#48887) (#49850)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 55974f7fb90 [fix](mtmv) Fix materialized rewrite oom when the num of relation mapping is too large (#48887) (#49850)
55974f7fb90 is described below
commit 55974f7fb90871cea17811938fafd4ceede23856
Author: seawinde <wu...@selectdb.com>
AuthorDate: Tue Apr 8 15:18:26 2025 +0800
[fix](mtmv) Fix materialized rewrite oom when the num of relation mapping is too large (#48887) (#49850)
---
.../mv/AbstractMaterializedViewRule.java | 11 +-
.../rules/exploration/mv/mapping/Mapping.java | 4 +-
.../exploration/mv/mapping/RelationMapping.java | 113 +++++---
.../doris/nereids/mv/RelationMappingTest.java | 158 ++++++++++++
.../rules/exploration/mv/HyperGraphAggTest.java | 3 +-
.../nereids/rules/exploration/mv/MappingTest.java | 287 ++++++++++++++++++++-
.../apache/doris/nereids/sqltest/SqlTestBase.java | 3 +-
.../mv/many_self_join/many_self_join.groovy | 92 +++++++
8 files changed, 624 insertions(+), 47 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 33acf462fb1..e8e46b68f3d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -180,21 +180,16 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
() -> String.format("matchMode is %s", matchMode));
return rewriteResults;
}
+ SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable();
+ int materializedViewRelationMappingMaxCount = sessionVariable.getMaterializedViewRelationMappingMaxCount();
List<RelationMapping> queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(),
- viewStructInfo.getRelations());
+ viewStructInfo.getRelations(), materializedViewRelationMappingMaxCount);
// if any relation in query and view can not map, bail out.
if (queryToViewTableMappings == null) {
materializationContext.recordFailReason(queryStructInfo,
"Query to view table mapping is null", () -> "");
return rewriteResults;
}
- SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable();
- int materializedViewRelationMappingMaxCount = sessionVariable.getMaterializedViewRelationMappingMaxCount();
- if (queryToViewTableMappings.size() > materializedViewRelationMappingMaxCount) {
- LOG.warn("queryToViewTableMappings is over limit and be intercepted");
- queryToViewTableMappings = queryToViewTableMappings.subList(0, materializedViewRelationMappingMaxCount);
- }
-
for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) {
SlotMapping queryToViewSlotMapping =
materializationContext.getSlotMappingFromCache(queryToViewTableMapping);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
index d9192569a0c..08b2c30275b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java
@@ -102,7 +102,9 @@ public abstract class Mapping {
@Override
public String toString() {
- return "MappedRelation{" + "relationId=" + relationId + ", slotNameToSlotMap=" + slotNameToSlotMap + '}';
+ String relationTableName = belongedRelation.getTable().getNameWithFullQualifiers();
+ return "MappedRelation{" + "relationId=" + relationId + ", relationTableName="
+ + relationTableName + ", slotNameToSlotMap=" + slotNameToSlotMap + '}';
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
index 42d4cd59b0d..3952db9b9af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
@@ -29,9 +29,11 @@ import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableBiMap.Builder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@@ -44,6 +46,7 @@ import java.util.Set;
*/
public class RelationMapping extends Mapping {
+ private static final Logger LOG = LogManager.getLogger(RelationMapping.class);
private final ImmutableBiMap<MappedRelation, MappedRelation> mappedRelationMap;
public RelationMapping(ImmutableBiMap<MappedRelation, MappedRelation> mappedRelationMap) {
@@ -61,7 +64,8 @@ public class RelationMapping extends Mapping {
/**
* Generate mapping according to source and target relation
*/
- public static List<RelationMapping> generate(List<CatalogRelation> sources, List<CatalogRelation> targets) {
+ public static List<RelationMapping> generate(List<CatalogRelation> sources, List<CatalogRelation> targets,
+ int relationMappingMaxCount) {
// Construct tmp map, key is the table qualifier, value is the corresponding catalog relations
HashMultimap<TableIdentifier, MappedRelation> sourceTableRelationIdMap = HashMultimap.create();
for (CatalogRelation relation : sources) {
@@ -92,39 +96,25 @@ public class RelationMapping extends Mapping {
}
// relation appear more than once, should cartesian them and power set to correct combination
// if query is select * from tableA0, tableA1, materialized view is select * from tableA2, tableA3,
- // tableA is the same table used by both query and materialized view
- // relationMapping will be
- // tableA0 tableA2
- // tableA0 tableA3
- // tableA1 tableA2
- // tableA1 tableA3
- ImmutableList<Pair<MappedRelation, MappedRelation>> relationMapping = Sets.cartesianProduct(
- sourceMappedRelations, targetMappedRelations)
- .stream()
- .map(listPair -> Pair.of(listPair.get(0), listPair.get(1)))
- .collect(ImmutableList.toImmutableList());
-
- // the mapping in relationMappingPowerList should be bi-direction
+ // the relationMappingPowerList in relationMappingPowerList should be bi-direction
// [
- // {tableA0 tableA2, tableA1 tableA3}
- // {tableA0 tableA3, tableA1 tableA2}
+ // {tableA0 -> tableA2, tableA1 -> tableA3}
+ // {tableA0 -> tableA3, tableA1 -> tableA2}
// ]
+ // query is select * from tableA0, tableA1, tableA4
List<BiMap<MappedRelation, MappedRelation>> relationMappingPowerList = new ArrayList<>();
- int relationMappingSize = relationMapping.size();
- int relationMappingMinSize = Math.min(sourceMappedRelations.size(), targetMappedRelations.size());
- for (int i = 0; i < relationMappingSize; i++) {
- HashBiMap<MappedRelation, MappedRelation> relationBiMap = HashBiMap.create();
- relationBiMap.put(relationMapping.get(i).key(), relationMapping.get(i).value());
- for (int j = i + 1; j < relationMappingSize; j++) {
- if (!relationBiMap.containsKey(relationMapping.get(j).key())
- && !relationBiMap.containsValue(relationMapping.get(j).value())) {
- relationBiMap.put(relationMapping.get(j).key(), relationMapping.get(j).value());
- }
- }
- // mapping should contain min num of relation in source or target at least
- if (relationBiMap.size() >= relationMappingMinSize) {
- relationMappingPowerList.add(relationBiMap);
+ List<Pair<MappedRelation[], MappedRelation[]>> combinations = getUniquePermutation(
+ sourceMappedRelations.toArray(new MappedRelation[0]),
+ targetMappedRelations.toArray(new MappedRelation[0]), relationMappingMaxCount);
+ for (Pair<MappedRelation[], MappedRelation[]> combination : combinations) {
+ BiMap<MappedRelation, MappedRelation> combinationBiMap = HashBiMap.create();
+ MappedRelation[] key = combination.key();
+ MappedRelation[] value = combination.value();
+ int length = Math.min(key.length, value.length);
+ for (int i = 0; i < length; i++) {
+ combinationBiMap.put(key[i], value[i]);
}
+ relationMappingPowerList.add(combinationBiMap);
}
mappedRelations.add(relationMappingPowerList);
}
@@ -167,4 +157,65 @@ public class RelationMapping extends Mapping {
public int hashCode() {
return Objects.hash(mappedRelationMap);
}
+
+ /**
+ * Permutation and remove duplicated element
+ * For example:
+ * Given [1, 4, 5] and [191, 194, 195]
+ * This would return
+ * [
+ * [(1, 191) (4, 194) (5, 195)],
+ * [(1, 191) (4, 195) (5, 194)],
+ * [(1, 194) (4, 191) (5, 195)],
+ * [(1, 194) (4, 195) (5, 191)],
+ * [(1, 195) (4, 191) (5, 194)],
+ * [(1, 195) (4, 194) (5, 191)]
+ * ]
+ * */
+ public static List<Pair<MappedRelation[], MappedRelation[]>> getUniquePermutation(
+ MappedRelation[] left, MappedRelation[] right, int maxMappingCount) {
+ boolean needSwap = left.length > right.length;
+ if (needSwap) {
+ MappedRelation[] temp = left;
+ left = right;
+ right = temp;
+ }
+
+ boolean[] used = new boolean[right.length];
+ MappedRelation[] current = new MappedRelation[left.length];
+ List<Pair<MappedRelation[], MappedRelation[]>> results = new ArrayList<>();
+ backtrack(left, right, 0, used, current, results, maxMappingCount);
+ if (needSwap) {
+ List<Pair<MappedRelation[], MappedRelation[]>> tmpResults = results;
+ results = new ArrayList<>();
+ for (Pair<MappedRelation[], MappedRelation[]> relation : tmpResults) {
+ results.add(Pair.of(relation.value(), relation.key()));
+ }
+ }
+ return results;
+ }
+
+ private static void backtrack(MappedRelation[] left, MappedRelation[] right, int index,
+ boolean[] used, MappedRelation[] current, List<Pair<MappedRelation[], MappedRelation[]>> results,
+ int maxMappingCount) {
+ if (results.size() >= maxMappingCount) {
+ LOG.warn("queryToViewTableMappings is over limit and be intercepted, "
+ + "results size is {}, MappedRelation left is {}, MappedRelation right is {}",
+ results.size(), Arrays.toString(left), Arrays.toString(right));
+ return;
+ }
+ if (index == left.length) {
+ results.add(Pair.of(left, Arrays.copyOf(current, current.length)));
+ return;
+ }
+
+ for (int i = 0; i < right.length; i++) {
+ if (!used[i]) {
+ used[i] = true;
+ current[index] = right[i];
+ backtrack(left, right, index + 1, used, current, results, maxMappingCount);
+ used[i] = false;
+ }
+ }
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationMappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationMappingTest.java
new file mode 100644
index 00000000000..35b110baf3b
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/RelationMappingTest.java
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.mv;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import mockit.Mocked;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class RelationMappingTest {
+
+ @Mocked
+ public CatalogRelation catalogRelation;
+
+ /**
+ * Permutation and remove duplicated element
+ * For example:
+ * Given [1, 4, 5] and [191, 194, 195]
+ * This would return
+ * [
+ * [(1, 191) (4, 194) (5, 195)],
+ * [(1, 191) (4, 195) (5, 194)],
+ * [(1, 194) (4, 191) (5, 195)],
+ * [(1, 194) (4, 195) (5, 191)],
+ * [(1, 195) (4, 191) (5, 194)],
+ * [(1, 195) (4, 194) (5, 191)]
+ * ]
+ * */
+ @Test
+ public void testGetPermutation() {
+
+ MappedRelation mr1 = MappedRelation.of(new RelationId(1), catalogRelation);
+ MappedRelation mr4 = MappedRelation.of(new RelationId(4), catalogRelation);
+ MappedRelation mr5 = MappedRelation.of(new RelationId(5), catalogRelation);
+
+ MappedRelation mr191 = MappedRelation.of(new RelationId(191), catalogRelation);
+ MappedRelation mr194 = MappedRelation.of(new RelationId(194), catalogRelation);
+ MappedRelation mr195 = MappedRelation.of(new RelationId(195), catalogRelation);
+
+ MappedRelation[] left = new MappedRelation[] {mr1, mr4, mr5};
+ MappedRelation[] right = new MappedRelation[] {mr191, mr194, mr195};
+
+ List<Pair<MappedRelation[], MappedRelation[]>> uniquePermutation = RelationMapping.getUniquePermutation(left,
+ right, 10);
+ Set<BiMap<Integer, Integer>> generatedMappedSet = generateDirectPermutations(uniquePermutation);
+
+ Set<BiMap<Integer, Integer>> exceptedSet = new HashSet<>();
+ BiMap<Integer, Integer> eachMap = HashBiMap.create();
+ eachMap.put(1, 195);
+ eachMap.put(4, 194);
+ eachMap.put(5, 191);
+ exceptedSet.add(eachMap);
+ eachMap = HashBiMap.create();
+ eachMap.put(1, 194);
+ eachMap.put(4, 191);
+ eachMap.put(5, 195);
+ exceptedSet.add(eachMap);
+ eachMap = HashBiMap.create();
+ eachMap.put(1, 194);
+ eachMap.put(4, 195);
+ eachMap.put(5, 191);
+ exceptedSet.add(eachMap);
+ eachMap = HashBiMap.create();
+ eachMap.put(1, 195);
+ eachMap.put(4, 191);
+ eachMap.put(5, 194);
+ exceptedSet.add(eachMap);
+ eachMap = HashBiMap.create();
+ eachMap.put(1, 191);
+ eachMap.put(4, 194);
+ eachMap.put(5, 195);
+ exceptedSet.add(eachMap);
+ eachMap = HashBiMap.create();
+ eachMap.put(1, 191);
+ eachMap.put(4, 195);
+ eachMap.put(5, 194);
+ exceptedSet.add(eachMap);
+
+ Assertions.assertEquals(exceptedSet, generatedMappedSet);
+ }
+
+ /**
+ * Permutation and remove duplicated element
+ * For example:
+ * Given [1, 4, 5] and [191, 194, 195]
+ * This would return
+ * [
+ * [(1, 191) (4, 194) (5, 195)],
+ * [(1, 191) (4, 195) (5, 194)],
+ * [(1, 194) (4, 191) (5, 195)],
+ * [(1, 194) (4, 195) (5, 191)],
+ * [(1, 195) (4, 191) (5, 194)],
+ * [(1, 195) (4, 194) (5, 191)]
+ * ]
+ * */
+ @Test
+ public void testGetPermutationWithLimit() {
+
+ MappedRelation mr1 = MappedRelation.of(new RelationId(1), catalogRelation);
+ MappedRelation mr4 = MappedRelation.of(new RelationId(4), catalogRelation);
+ MappedRelation mr5 = MappedRelation.of(new RelationId(5), catalogRelation);
+
+ MappedRelation mr191 = MappedRelation.of(new RelationId(191), catalogRelation);
+ MappedRelation mr194 = MappedRelation.of(new RelationId(194), catalogRelation);
+ MappedRelation mr195 = MappedRelation.of(new RelationId(195), catalogRelation);
+
+ MappedRelation[] left = new MappedRelation[] {mr1, mr4, mr5};
+ MappedRelation[] right = new MappedRelation[] {mr191, mr194, mr195};
+
+ List<Pair<MappedRelation[], MappedRelation[]>> uniquePermutation = RelationMapping.getUniquePermutation(left,
+ right, 5);
+
+ Assertions.assertEquals(5, uniquePermutation.size());
+ }
+
+ private Set<BiMap<Integer, Integer>> generateDirectPermutations(
+ List<Pair<MappedRelation[], MappedRelation[]>> uniquePermutation) {
+ Set<BiMap<Integer, Integer>> relationMappingPowerSet = new HashSet<>();
+ for (Pair<MappedRelation[], MappedRelation[]> combination : uniquePermutation) {
+ BiMap<Integer, Integer> combinationBiMap = HashBiMap.create();
+ MappedRelation[] key = combination.key();
+ MappedRelation[] value = combination.value();
+ int length = Math.min(key.length, value.length);
+ for (int i = 0; i < length; i++) {
+ combinationBiMap.put(key[i].getRelationId().asInt(), value[i].getRelationId().asInt());
+ }
+ relationMappingPowerSet.add(combinationBiMap);
+ }
+ return relationMappingPowerSet;
+ }
+
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
index 44939cc61c6..b62c41db00d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphAggTest.java
@@ -93,7 +93,8 @@ class HyperGraphAggTest extends SqlTestBase {
null, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, p2,
null, new BitSet()).get(0);
- RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0);
+ RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations(), 8)
+ .get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
index 4371b9f5785..509acfba207 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
@@ -33,7 +33,9 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* MappingTest
@@ -105,7 +107,8 @@ public class MappingTest extends TestWithFeService {
List<CatalogRelation> targetRelations = new ArrayList<>();
targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
- List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations);
+ List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations,
+ 8);
Assertions.assertNotNull(generateRelationMapping);
Assertions.assertEquals(1, generateRelationMapping.size());
@@ -154,7 +157,8 @@ public class MappingTest extends TestWithFeService {
List<CatalogRelation> targetRelations = new ArrayList<>();
targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
- List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations);
+ List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations,
+ 8);
Assertions.assertNotNull(generateRelationMapping);
Assertions.assertEquals(1, generateRelationMapping.size());
@@ -200,7 +204,8 @@ public class MappingTest extends TestWithFeService {
List<CatalogRelation> targetRelations = new ArrayList<>();
targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
- List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations);
+ List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations,
+ 8);
Assertions.assertNotNull(generateRelationMapping);
Assertions.assertEquals(1, generateRelationMapping.size());
@@ -246,7 +251,8 @@ public class MappingTest extends TestWithFeService {
List<CatalogRelation> targetRelations = new ArrayList<>();
targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
- List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations);
+ List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations,
+ 8);
Assertions.assertNotNull(generateRelationMapping);
Assertions.assertEquals(2, generateRelationMapping.size());
@@ -306,7 +312,8 @@ public class MappingTest extends TestWithFeService {
List<CatalogRelation> targetRelations = new ArrayList<>();
targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
- List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations);
+ List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations,
+ 8);
Assertions.assertNotNull(generateRelationMapping);
Assertions.assertEquals(2, generateRelationMapping.size());
@@ -343,6 +350,262 @@ public class MappingTest extends TestWithFeService {
assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping);
}
+ // Test more than two tables, and the same table num in source plan is equals to target plan
+ @Test
+ public void testGenerateMapping6() {
+ Plan sourcePlan = PlanChecker.from(connectContext)
+ .analyze("SELECT orders.*, l1.l_orderkey, l2.l_orderkey, l3.l_orderkey "
+ + "FROM\n"
+ + " orders,\n"
+ + " lineitem l1,\n"
+ + " lineitem l2,\n"
+ + " lineitem l3\n"
+ + "WHERE\n"
+ + " l1.l_orderkey = l2.l_orderkey and l1.l_orderkey = l3.l_orderkey\n"
+ + " AND l1.l_orderkey = o_orderkey")
+ .getPlan();
+
+ Plan targetPlan = PlanChecker.from(connectContext)
+ .analyze("SELECT orders.*, l1.l_orderkey, l2.l_orderkey, l3.l_orderkey "
+ + "FROM\n"
+ + " lineitem l1,\n"
+ + " orders,\n"
+ + " lineitem l2,\n"
+ + " lineitem l3\n"
+ + "WHERE\n"
+ + " l1.l_orderkey = l2.l_orderkey and l1.l_orderkey = l3.l_orderkey\n"
+ + " AND l2.l_orderkey = o_orderkey")
+ .getPlan();
+ List<CatalogRelation> sourceRelations = new ArrayList<>();
+ sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations);
+
+ List<CatalogRelation> targetRelations = new ArrayList<>();
+ targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
+
+ List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations,
+ 8);
+ Assertions.assertNotNull(generateRelationMapping);
+ Assertions.assertEquals(6, generateRelationMapping.size());
+
+ // expected table relation mapping is as following
+ // (1, 3), (2, 2), (3, 0), (0, 1)
+ // (1, 0), (2, 3), (3, 2), (0, 1)
+ // (1, 2), (2, 3), (3, 0), (0, 1)
+ // (1, 0), (2, 2), (3, 3), (0, 1)
+ // (1, 2), (2, 0), (3, 3), (0, 1)
+ // (1, 3), (2, 0), (3, 2), (0, 1)
+ Set<BiMap<RelationId, RelationId>> expectedRelationMappingSet = new HashSet<>();
+ BiMap<RelationId, RelationId> expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(3), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(3), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(3), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(3), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(3), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(3), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ assertRelationMapping(new HashSet<>(generateRelationMapping), expectedRelationMappingSet);
+ }
+
+ // Test more than two tables, and the same table num in source plan is less then the num of target plan
+ @Test
+ public void testGenerateMapping7() {
+ Plan sourcePlan = PlanChecker.from(connectContext)
+ .analyze("SELECT orders.*, l1.l_orderkey, l3.l_orderkey "
+ + "FROM\n"
+ + " orders,\n"
+ + " lineitem l1,\n"
+ + " lineitem l3\n"
+ + "WHERE\n"
+ + " l1.l_orderkey = l3.l_orderkey\n"
+ + " AND l1.l_orderkey = o_orderkey")
+ .getPlan();
+
+ Plan targetPlan = PlanChecker.from(connectContext)
+ .analyze("SELECT orders.*, l1.l_orderkey, l2.l_orderkey, l3.l_orderkey "
+ + "FROM\n"
+ + " lineitem l1,\n"
+ + " orders,\n"
+ + " lineitem l2,\n"
+ + " lineitem l3\n"
+ + "WHERE\n"
+ + " l1.l_orderkey = l2.l_orderkey and l1.l_orderkey = l3.l_orderkey\n"
+ + " AND l2.l_orderkey = o_orderkey")
+ .getPlan();
+ List<CatalogRelation> sourceRelations = new ArrayList<>();
+ sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations);
+
+ List<CatalogRelation> targetRelations = new ArrayList<>();
+ targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
+
+ List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations,
+ 8);
+ Assertions.assertNotNull(generateRelationMapping);
+ Assertions.assertEquals(6, generateRelationMapping.size());
+
+ // expected table relation mapping is as following
+ // (1, 2), (2, 0), (0, 1)
+ // (1, 2), (2, 3), (0, 1)
+ // (1, 0), (2, 2), (0, 1)
+ // (1, 0), (2, 3), (0, 1)
+ // (1, 3), (2, 0), (0, 1)
+ // (1, 3), (2, 2), (0, 1)
+ Set<BiMap<RelationId, RelationId>> expectedRelationMappingSet = new HashSet<>();
+ BiMap<RelationId, RelationId> expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(3));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ assertRelationMapping(new HashSet<>(generateRelationMapping), expectedRelationMappingSet);
+ }
+
+ // Test more than two tables, and the same table num in source plan is more then the num of target plan
+ @Test
+ public void testGenerateMapping8() {
+ Plan sourcePlan = PlanChecker.from(connectContext)
+ .analyze("SELECT orders.*, l1.l_orderkey, l2.l_orderkey, l3.l_orderkey "
+ + "FROM\n"
+ + " orders,\n"
+ + " lineitem l1,\n"
+ + " lineitem l2,\n"
+ + " lineitem l3\n"
+ + "WHERE\n"
+ + " l1.l_orderkey = l2.l_orderkey and l1.l_orderkey = l3.l_orderkey\n"
+ + " AND l1.l_orderkey = o_orderkey")
+ .getPlan();
+
+ Plan targetPlan = PlanChecker.from(connectContext)
+ .analyze("SELECT orders.*, l1.l_orderkey, l3.l_orderkey "
+ + "FROM\n"
+ + " lineitem l1,\n"
+ + " orders,\n"
+ + " lineitem l3\n"
+ + "WHERE\n"
+ + " l1.l_orderkey = l3.l_orderkey\n"
+ + " AND l3.l_orderkey = o_orderkey")
+ .getPlan();
+ List<CatalogRelation> sourceRelations = new ArrayList<>();
+ sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations);
+
+ List<CatalogRelation> targetRelations = new ArrayList<>();
+ targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
+
+ List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations,
+ 8);
+ Assertions.assertNotNull(generateRelationMapping);
+ Assertions.assertEquals(6, generateRelationMapping.size());
+
+ // expected table relation mapping is as following
+ // (1, 0), (2, 2), (0, 1)
+ // (1, 0), (3, 2), (0, 1)
+ // (2, 0), (1, 2), (0, 1)
+ // (3, 0), (2, 2), (0, 1)
+ // (2, 0), (3, 2), (0, 1)
+ // (3, 0), (1, 2), (0, 1)
+ Set<BiMap<RelationId, RelationId>> expectedRelationMappingSet = new HashSet<>();
+ BiMap<RelationId, RelationId> expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(1), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(3), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(2), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(1), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(3), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(2), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(2), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(3), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ expectedRelationMapping = HashBiMap.create();
+ expectedRelationMapping.put(new RelationId(3), new RelationId(0));
+ expectedRelationMapping.put(new RelationId(1), new RelationId(2));
+ expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+ expectedRelationMappingSet.add(expectedRelationMapping);
+
+ assertRelationMapping(new HashSet<>(generateRelationMapping), expectedRelationMappingSet);
+ }
+
private void assertRelationMapping(RelationMapping relationMapping,
BiMap<RelationId, RelationId> expectRelationMapping,
BiMap<ExprId, ExprId> expectSlotMapping) {
@@ -362,6 +625,20 @@ public class MappingTest extends TestWithFeService {
Assertions.assertEquals(generatedSlotMapping, expectSlotMapping);
}
+ private void assertRelationMapping(Set<RelationMapping> relationMapping,
+ Set<BiMap<RelationId, RelationId>> expectRelationMapping) {
+ // check relation mapping if equals or not
+ Set<BiMap<RelationId, RelationId>> relationMappingSet = new HashSet<>();
+ relationMapping.forEach(mapping -> {
+ BiMap<RelationId, RelationId> generatedRelationMapping = HashBiMap.create();
+ mapping.getMappedRelationMap().forEach((key, value) ->
+ generatedRelationMapping.put(key.getRelationId(), value.getRelationId()));
+ relationMappingSet.add(generatedRelationMapping);
+ }
+ );
+ Assertions.assertEquals(relationMappingSet, expectRelationMapping);
+ }
+
protected static class RelationCollector extends DefaultPlanVisitor<Void, List<CatalogRelation>> {
public static final RelationCollector INSTANCE = new RelationCollector();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
index 9ebbc22a85c..46e121d167c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
@@ -108,7 +108,8 @@ public abstract class SqlTestBase extends TestWithFeService implements MemoPatte
context, new BitSet()).get(0);
StructInfo st2 = MaterializedViewUtils.extractStructInfo(p2, p2,
context, new BitSet()).get(0);
- RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0);
+ RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations(), 8)
+ .get(0);
SlotMapping sm = SlotMapping.generate(rm);
return LogicalCompatibilityContext.from(rm, sm, st1, st2);
}
diff --git a/regression-test/suites/nereids_rules_p0/mv/many_self_join/many_self_join.groovy b/regression-test/suites/nereids_rules_p0/mv/many_self_join/many_self_join.groovy
new file mode 100644
index 00000000000..67e558dc5d8
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/many_self_join/many_self_join.groovy
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("many_self_join") {
+
+ def db_name = context.config.getDbNameByFile(context.file)
+ def tb_name = db_name + "_tb"
+ def mv_name = db_name + "_mtmv"
+ sql """drop table if exists ${tb_name}"""
+ sql """
+ CREATE TABLE ${tb_name} (
+ employee_id INT,
+ employee_name VARCHAR(100),
+ manager_id INT
+ ) ENGINE=OLAP
+ DUPLICATE KEY(employee_id)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`employee_id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+ sql """
+ INSERT INTO ${tb_name} (employee_id, employee_name, manager_id) VALUES
+ (1, 'CEO', 1),
+ (2, 'Manager1', 1),
+ (3, 'Manager2', 1)
+ """
+
+ sql """drop MATERIALIZED VIEW if exists ${mv_name};"""
+ def mtmv_sql = """SELECT
+ e1.employee_name AS employee,
+ e2.employee_name AS manager_1,
+ e3.employee_name AS manager_2,
+ e4.employee_name AS manager_3,
+ e5.employee_name AS manager_4,
+ e6.employee_name AS manager_5,
+ e7.employee_name AS manager_6,
+ e8.employee_name AS manager_7,
+ e9.employee_name AS manager_8,
+ e10.employee_name AS manager_9
+ FROM
+ ${tb_name} e1
+ LEFT JOIN
+ ${tb_name} e2 ON e1.manager_id = e2.employee_id
+ LEFT JOIN
+ ${tb_name} e3 ON e2.manager_id = e3.employee_id
+ LEFT JOIN
+ ${tb_name} e4 ON e3.manager_id = e4.employee_id
+ LEFT JOIN
+ ${tb_name} e5 ON e4.manager_id = e5.employee_id
+ LEFT JOIN
+ ${tb_name} e6 ON e5.manager_id = e6.employee_id
+ LEFT JOIN
+ ${tb_name} e7 ON e6.manager_id = e7.employee_id
+ LEFT JOIN
+ ${tb_name} e8 ON e7.manager_id = e8.employee_id
+ LEFT JOIN
+ ${tb_name} e9 ON e8.manager_id = e9.employee_id
+ LEFT JOIN
+ ${tb_name} e10 ON e9.manager_id = e10.employee_id"""
+
+ create_async_mv(db_name, mv_name, mtmv_sql)
+
+ def mark = true
+ sql """USE ${db_name}"""
+ sql """set enable_nereids_timeout = true;"""
+ try {
+ mv_rewrite_success(mtmv_sql, mv_name)
+ } catch (Exception e) {
+ log.info("e.getMessage(): " + e.getMessage())
+ mark = false
+ }
+
+ assertTrue(mark)
+
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org