You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/04/24 15:38:44 UTC
(doris) 01/03: [fix](mtmv) Fix exception when create materialized view with cte (#33988)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ec0235d074d5173bf737dfc357e479a95223c066
Author: seawinde <14...@users.noreply.github.com>
AuthorDate: Wed Apr 24 22:11:30 2024 +0800
[fix](mtmv) Fix exception when create materialized view with cte (#33988)
Fix exception when create materialized view with cte, after this fix, can create materialized view with following
```
CREATE MATERIALIZED VIEW mv_with_cte
BUILD IMMEDIATE REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES ('replication_num' = '1')
AS
with `test_with` AS (
select l_partkey, l_suppkey
from lineitem
union
select
ps_partkey, ps_suppkey
from
partsupp)
select * from test_with;
```
this is brought from https://github.com/apache/doris/pull/28144
---
.../plans/commands/UpdateMvByPartitionCommand.java | 39 ++++++++++++++++++-
regression-test/data/mtmv_p0/test_build_mtmv.out | 6 +++
.../suites/mtmv_p0/test_build_mtmv.groovy | 44 ++++++++++++++++++++++
3 files changed, 88 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
index d63aee6ea70..ac8db7f9762 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
@@ -41,10 +41,12 @@ import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Sink;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.RelationUtil;
@@ -197,11 +199,21 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
}
/**
- * Add predicates on base table when mv can partition update
+ * Add predicates on base table when mv can partition update, Also support plan that contain cte and view
*/
public static class PredicateAdder extends DefaultPlanRewriter<Map<TableIf, Set<Expression>>> {
+
+ // record view and cte name parts, these should be ignored and visit it's actual plan
+ public Set<List<String>> virtualRelationNamePartSet = new HashSet<>();
+
@Override
public Plan visitUnboundRelation(UnboundRelation unboundRelation, Map<TableIf, Set<Expression>> predicates) {
+ if (predicates.isEmpty()) {
+ return unboundRelation;
+ }
+ if (virtualRelationNamePartSet.contains(unboundRelation.getNameParts())) {
+ return unboundRelation;
+ }
List<String> tableQualifier = RelationUtil.getQualifierName(ConnectContext.get(),
unboundRelation.getNameParts());
TableIf table = RelationUtil.getTable(tableQualifier, Env.getCurrentEnv());
@@ -212,9 +224,34 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
return unboundRelation;
}
+ @Override
+ public Plan visitLogicalCTE(LogicalCTE<? extends Plan> cte, Map<TableIf, Set<Expression>> predicates) {
+ if (predicates.isEmpty()) {
+ return cte;
+ }
+ for (LogicalSubQueryAlias<Plan> subQueryAlias : cte.getAliasQueries()) {
+ this.virtualRelationNamePartSet.add(subQueryAlias.getQualifier());
+ subQueryAlias.children().forEach(subQuery -> subQuery.accept(this, predicates));
+ }
+ return super.visitLogicalCTE(cte, predicates);
+ }
+
+ @Override
+ public Plan visitLogicalSubQueryAlias(LogicalSubQueryAlias<? extends Plan> subQueryAlias,
+ Map<TableIf, Set<Expression>> predicates) {
+ if (predicates.isEmpty()) {
+ return subQueryAlias;
+ }
+ this.virtualRelationNamePartSet.add(subQueryAlias.getQualifier());
+ return super.visitLogicalSubQueryAlias(subQueryAlias, predicates);
+ }
+
@Override
public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation,
Map<TableIf, Set<Expression>> predicates) {
+ if (predicates.isEmpty()) {
+ return catalogRelation;
+ }
TableIf table = catalogRelation.getTable();
if (predicates.containsKey(table)) {
return new LogicalFilter<>(ImmutableSet.of(ExpressionUtils.or(predicates.get(table))),
diff --git a/regression-test/data/mtmv_p0/test_build_mtmv.out b/regression-test/data/mtmv_p0/test_build_mtmv.out
index 4c2b4ea7529..db69c393748 100644
--- a/regression-test/data/mtmv_p0/test_build_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_build_mtmv.out
@@ -63,3 +63,9 @@ zhangsang 200
-- !desc_mv --
field_1 VARCHAR(16) No false \N NONE
+-- !query_mv_with_cte --
+2 3
+2 4
+3 3
+4 3
+
diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
index c431c3a7885..34e11608b3f 100644
--- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
@@ -625,6 +625,30 @@ suite("test_build_mtmv") {
(5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
"""
+ sql """
+ drop table if exists partsupp
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS partsupp (
+ ps_partkey INTEGER NOT NULL,
+ ps_suppkey INTEGER NOT NULL,
+ ps_availqty INTEGER NOT NULL,
+ ps_supplycost DECIMALV3(15,2) NOT NULL,
+ ps_comment VARCHAR(199) NOT NULL
+ )
+ DUPLICATE KEY(ps_partkey, ps_suppkey)
+ DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ )"""
+
+ sql """
+ insert into partsupp values
+ (2, 3, 9, 10.01, 'supply1'),
+ (2, 3, 10, 11.01, 'supply2');
+ """
+
sql """DROP MATERIALIZED VIEW IF EXISTS test_varchar_literal_mv;"""
sql """
CREATE MATERIALIZED VIEW test_varchar_literal_mv
@@ -635,4 +659,24 @@ suite("test_build_mtmv") {
select case when l_orderkey > 1 then "一二三四" else "五六七八" end as field_1 from lineitem;
"""
qt_desc_mv """desc test_varchar_literal_mv;"""
+
+ sql """DROP MATERIALIZED VIEW IF EXISTS mv_with_cte;"""
+ sql """
+ CREATE MATERIALIZED VIEW mv_with_cte
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ with `test_with` AS (
+ select l_partkey, l_suppkey
+ from lineitem
+ union
+ select
+ ps_partkey, ps_suppkey
+ from
+ partsupp)
+ select * from test_with;
+ """
+ waitingMTMVTaskFinished(getJobName("regression_test_mtmv_p0", "mv_with_cte"))
+ order_qt_query_mv_with_cte """select * from mv_with_cte;"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org