You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2025/11/18 02:57:36 UTC
(doris) branch branch-4.0 updated: branch-4.0:[feature](mtmv)create mtmv can use view (#56423) (#58010)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new c8f7784acac branch-4.0:[feature](mtmv)create mtmv can use view (#56423) (#58010)
c8f7784acac is described below
commit c8f7784acac7f3aca533d060de25a50a62074e2f
Author: zhangdong <zh...@selectdb.com>
AuthorDate: Tue Nov 18 10:57:29 2025 +0800
branch-4.0:[feature](mtmv)create mtmv can use view (#56423) (#58010)
cherry-pick: https://github.com/apache/doris/pull/56423
---
.../main/java/org/apache/doris/alter/Alter.java | 9 +-
.../main/java/org/apache/doris/catalog/MTMV.java | 34 ++-
.../apache/doris/datasource/InternalCatalog.java | 9 +-
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 63 ++---
.../MTMVAnalyzeQueryInfo.java} | 34 ++-
.../org/apache/doris/mtmv/MTMVHookService.java | 14 ++
.../java/org/apache/doris/mtmv/MTMVJobManager.java | 9 +
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 19 ++
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 8 +-
.../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 261 ++++++++++++++++++++-
.../doris/mtmv/MTMVRefreshPartitionSnapshot.java | 6 +-
.../java/org/apache/doris/mtmv/MTMVRelation.java | 34 ++-
.../org/apache/doris/mtmv/MTMVRelationManager.java | 99 ++++++--
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 2 +-
.../java/org/apache/doris/mtmv/MTMVService.java | 18 +-
.../main/java/org/apache/doris/mtmv/MTMVUtil.java | 10 +-
.../org/apache/doris/nereids/CascadesContext.java | 4 +-
.../org/apache/doris/nereids/NereidsPlanner.java | 2 +-
.../org/apache/doris/nereids/StatementContext.java | 23 ++
.../executor/TableCollectAndHookInitializer.java | 16 +-
.../rules/analysis/CollectOneLevelRelation.java | 4 +-
.../nereids/rules/analysis/CollectRelation.java | 19 +-
.../exploration/mv/MaterializedViewUtils.java | 13 +-
.../plans/commands/UpdateMvByPartitionCommand.java | 6 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 174 +-------------
.../trees/plans/logical/LogicalCheckPolicy.java | 17 ++
.../doris/tablefunction/MetadataGenerator.java | 29 ++-
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 2 +-
.../org/apache/doris/mtmv/MTMVPlanUtilTest.java | 229 +++++++++++++++++-
.../apache/doris/mtmv/MTMVRelationManagerTest.java | 42 ++--
.../org/apache/doris/mtmv/MTMVRelationTest.java | 117 +++++++++
.../apache/doris/mtmv/MTMVSchemaChangeTest.java | 94 ++++++++
.../java/org/apache/doris/mtmv/MTMVTaskTest.java | 3 +-
.../test/java/org/apache/doris/mtmv/MTMVTest.java | 5 +-
.../org/apache/doris/nereids/util/PlanChecker.java | 6 +-
.../apache/doris/utframe/TestWithFeService.java | 7 +
.../data/mtmv_p0/test_create_mtmv_with_view.out | 30 +++
.../mtmv_p0/test_create_mtmv_with_view_alter.out | 19 ++
.../test_create_mtmv_with_view_alter_table.out | 19 ++
.../mtmv_p0/test_create_mtmv_with_view_commit.out | 7 +
.../mtmv_p0/test_create_mtmv_with_view_cte.out | 7 +
.../mtmv_p0/test_create_mtmv_with_view_mtmv.out | 20 ++
.../mtmv_p0/test_create_mtmv_with_view_pct.out | 21 ++
.../mtmv_p0/test_create_mtmv_with_view_rollup.out | 9 +
.../mtmv_p0/test_limit_partition_view_mtmv.out | 17 ++
.../data/mtmv_p0/test_view_hive_mtmv.out | 28 +++
.../suites/mtmv_p0/test_build_mtmv.groovy | 15 --
.../mtmv_p0/test_create_mtmv_with_view.groovy | 112 +++++++++
.../test_create_mtmv_with_view_alter.groovy | 94 ++++++++
.../test_create_mtmv_with_view_alter_table.groovy | 103 ++++++++
.../test_create_mtmv_with_view_commit.groovy | 69 ++++++
.../mtmv_p0/test_create_mtmv_with_view_cte.groovy | 92 ++++++++
.../mtmv_p0/test_create_mtmv_with_view_mtmv.groovy | 126 ++++++++++
.../mtmv_p0/test_create_mtmv_with_view_pct.groovy | 103 ++++++++
.../test_create_mtmv_with_view_rollup.groovy | 81 +++++++
.../mtmv_p0/test_limit_partition_view_mtmv.groovy | 254 ++++++++++++++++++++
.../suites/mtmv_p0/test_view_hive_mtmv.groovy | 98 ++++++++
57 files changed, 2412 insertions(+), 353 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 12b1fedcd67..aad6ab45479 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -826,6 +826,7 @@ public class Alter {
db.registerTable(view);
AlterViewInfo alterViewInfo = new AlterViewInfo(db.getId(), view.getId(),
inlineViewDef, newFullSchema, sqlMode, comment);
+ Env.getCurrentEnv().getMtmvService().alterView(new BaseTableInfo(view));
Env.getCurrentEnv().getEditLog().logModifyViewDef(alterViewInfo);
LOG.info("modify view[{}] definition to {}", viewName, inlineViewDef);
} finally {
@@ -863,7 +864,7 @@ public class Alter {
db.unregisterTable(viewName);
db.registerTable(view);
-
+ Env.getCurrentEnv().getMtmvService().alterView(new BaseTableInfo(view));
LOG.info("replay modify view[{}] definition to {}", viewName, inlineViewDef);
} finally {
view.writeUnlock();
@@ -1236,6 +1237,7 @@ public class Alter {
public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) {
TableNameInfo tbl = alterMTMV.getMvName();
MTMV mtmv = null;
+ boolean alterSuccess = true;
try {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(tbl.getDb());
mtmv = (MTMV) db.getTableOrMetaException(tbl.getTbl(), TableType.MATERIALIZED_VIEW);
@@ -1250,7 +1252,8 @@ public class Alter {
mtmv.alterMvProperties(alterMTMV.getMvProperties());
break;
case ADD_TASK:
- mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(), alterMTMV.getPartitionSnapshots(),
+ alterSuccess = mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(),
+ alterMTMV.getPartitionSnapshots(),
isReplay);
// If it is not a replay thread, it means that the current service is already a new version
// and does not require compatibility
@@ -1265,7 +1268,7 @@ public class Alter {
Env.getCurrentEnv().getMtmvService().alterJob(mtmv, isReplay);
}
// 4. log it and replay it in the follower
- if (!isReplay) {
+ if (!isReplay && alterSuccess) {
Env.getCurrentEnv().getEditLog().logAlterMTMV(alterMTMV);
}
} catch (UserException e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index 14571133e0d..b4791bf06a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -84,6 +84,7 @@ public class MTMV extends OlapTable {
private MTMVRefreshSnapshot refreshSnapshot;
// Should update after every fresh, not persist
private MTMVCache cache;
+ private long schemaChangeVersion;
// For deserialization
public MTMV() {
@@ -179,13 +180,26 @@ public class MTMV extends OlapTable {
writeMvLock();
try {
// only can update state, refresh state will be change by add task
+ this.schemaChangeVersion++;
return this.status.updateStateAndDetail(newStatus);
} finally {
writeMvUnlock();
}
}
- public void addTaskResult(MTMVTask task, MTMVRelation relation,
+ public void processBaseViewChange(String schemaChangeDetail) {
+ writeMvLock();
+ try {
+ this.schemaChangeVersion++;
+ this.status.setState(MTMVState.SCHEMA_CHANGE);
+ this.status.setSchemaChangeDetail(schemaChangeDetail);
+ this.refreshSnapshot = new MTMVRefreshSnapshot();
+ } finally {
+ writeMvUnlock();
+ }
+ }
+
+ public boolean addTaskResult(MTMVTask task, MTMVRelation relation,
Map<String, MTMVRefreshPartitionSnapshot> partitionSnapshots, boolean isReplay) {
MTMVCache mtmvCache = null;
boolean needUpdateCache = false;
@@ -209,6 +223,14 @@ public class MTMV extends OlapTable {
}
writeMvLock();
try {
+ if (!isReplay && task.getMtmvSchemaChangeVersion() != this.schemaChangeVersion) {
+ LOG.warn(
+ "addTaskResult failed, schemaChangeVersion has changed. "
+ + "mvName: {}, taskId: {}, taskSchemaChangeVersion: {}, "
+ + "mvSchemaChangeVersion: {}",
+ name, task.getTaskId(), task.getMtmvSchemaChangeVersion(), this.schemaChangeVersion);
+ return false;
+ }
if (task.getStatus() == TaskStatus.SUCCESS) {
this.status.setState(MTMVState.NORMAL);
this.status.setSchemaChangeDetail(null);
@@ -224,6 +246,7 @@ public class MTMV extends OlapTable {
this.refreshSnapshot.updateSnapshots(partitionSnapshots, getPartitionNames());
Env.getCurrentEnv().getMtmvService()
.refreshComplete(this, relation, task);
+ return true;
} finally {
writeMvUnlock();
}
@@ -372,6 +395,15 @@ public class MTMV extends OlapTable {
return refreshSnapshot;
}
+ public long getSchemaChangeVersion() {
+ readMvLock();
+ try {
+ return schemaChangeVersion;
+ } finally {
+ readMvUnlock();
+ }
+ }
+
/**
* generateMvPartitionDescs
*
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 4868caefd5e..ebfccd631d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -121,6 +121,7 @@ import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.es.EsRepository;
import org.apache.doris.event.DropPartitionEvent;
+import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType;
@@ -989,8 +990,9 @@ public class InternalCatalog implements CatalogIf<Database> {
} finally {
table.writeUnlock();
}
-
- Env.getCurrentEnv().getMtmvService().dropTable(table);
+ if (table instanceof OlapTable) {
+ Env.getCurrentEnv().getMtmvService().dropTable(table);
+ }
if (Config.isCloudMode()) {
((CloudGlobalTransactionMgr) Env.getCurrentGlobalTransactionMgr())
.clearTableLastTxnId(db.getId(), table.getId());
@@ -1017,6 +1019,9 @@ public class InternalCatalog implements CatalogIf<Database> {
if (table instanceof MTMV) {
Env.getCurrentEnv().getMtmvService().dropJob((MTMV) table, isReplay);
}
+ if (table instanceof View) {
+ Env.getCurrentEnv().getMtmvService().dropView(new BaseTableInfo(table));
+ }
Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId());
Env.getCurrentEnv().getDictionaryManager().dropTableDictionaries(db.getName(), table.getName());
Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), table.getId());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index a92754e8aee..d99d560ae51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -24,7 +24,6 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@@ -59,7 +58,6 @@ import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand;
-import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
import org.apache.doris.qe.AuditLogHelper;
import org.apache.doris.qe.ConnectContext;
@@ -90,7 +88,6 @@ import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
public class MTMVTask extends AbstractTask {
private static final Logger LOG = LogManager.getLogger(MTMVTask.class);
@@ -158,6 +155,7 @@ public class MTMVTask extends AbstractTask {
private MTMVRelation relation;
private StmtExecutor executor;
private Map<String, MTMVRefreshPartitionSnapshot> partitionSnapshots;
+ private long mtmvSchemaChangeVersion;
private Map<MvccTableInfo, MvccSnapshot> snapshots = Maps.newHashMap();
@@ -182,6 +180,7 @@ public class MTMVTask extends AbstractTask {
if (LOG.isDebugEnabled()) {
LOG.debug("mtmv task run, taskId: {}", super.getTaskId());
}
+ mtmvSchemaChangeVersion = mtmv.getSchemaChangeVersion();
ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK);
try {
if (LOG.isDebugEnabled()) {
@@ -193,10 +192,10 @@ public class MTMVTask extends AbstractTask {
}
// Every time a task is run, the relation is regenerated because baseTables and baseViews may change,
// such as deleting a table and creating a view with the same name
- Set<TableIf> tablesInPlan = MTMVPlanUtil.getBaseTableFromQuery(mtmv.getQuerySql(), ctx);
- this.relation = MTMVPlanUtil.generateMTMVRelation(tablesInPlan, ctx);
+ Pair<Set<TableIf>, Set<TableIf>> tablesInPlan = MTMVPlanUtil.getBaseTableFromQuery(mtmv.getQuerySql(), ctx);
+ this.relation = MTMVPlanUtil.generateMTMVRelation(tablesInPlan.first, tablesInPlan.second);
beforeMTMVRefresh();
- List<TableIf> tableIfs = Lists.newArrayList(tablesInPlan);
+ List<TableIf> tableIfs = Lists.newArrayList(tablesInPlan.first);
tableIfs.sort(Comparator.comparing(TableIf::getId));
MTMVRefreshContext context;
@@ -207,7 +206,7 @@ public class MTMVTask extends AbstractTask {
// if mtmv is schema_change, check if column type has changed
// If it's not in the schema_change state, the column type definitely won't change.
if (MTMVState.SCHEMA_CHANGE.equals(mtmv.getStatus().getState())) {
- checkColumnTypeIfChange(mtmv, ctx);
+ MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, ctx);
}
if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) {
MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable();
@@ -254,7 +253,8 @@ public class MTMVTask extends AbstractTask {
.subList(start, Math.min(end, needRefreshPartitions.size())));
// need get names before exec
Map<String, MTMVRefreshPartitionSnapshot> execPartitionSnapshots = MTMVPartitionUtil
- .generatePartitionSnapshots(context, relation.getBaseTablesOneLevel(), execPartitionNames);
+ .generatePartitionSnapshots(context, relation.getBaseTablesOneLevelAndFromView(),
+ execPartitionNames);
try {
executeWithRetry(execPartitionNames, tableWithPartKey);
} catch (Exception e) {
@@ -277,39 +277,6 @@ public class MTMVTask extends AbstractTask {
}
}
- private void checkColumnTypeIfChange(MTMV mtmv, ConnectContext ctx) throws JobException {
- List<ColumnDefinition> currentColumnsDefinition = MTMVPlanUtil.generateColumnsBySql(mtmv.getQuerySql(), ctx,
- mtmv.getMvPartitionInfo().getPartitionCol(),
- mtmv.getDistributionColumnNames(), null, mtmv.getTableProperty().getProperties());
- List<Column> currentColumns = currentColumnsDefinition.stream()
- .map(ColumnDefinition::translateToCatalogStyle)
- .collect(Collectors.toList());
- List<Column> originalColumns = mtmv.getBaseSchema(true);
- if (currentColumns.size() != originalColumns.size()) {
- throw new JobException(String.format(
- "column length not equals, please check whether columns of base table have changed, "
- + "original length is: %s, current length is: %s",
- originalColumns.size(), currentColumns.size()));
- }
- for (int i = 0; i < originalColumns.size(); i++) {
- if (!isTypeLike(originalColumns.get(i).getType(), currentColumns.get(i).getType())) {
- throw new JobException(String.format(
- "column type not same, please check whether columns of base table have changed, "
- + "column name is: %s, original type is: %s, current type is: %s",
- originalColumns.get(i).getName(), originalColumns.get(i).getType().toSql(),
- currentColumns.get(i).getType().toSql()));
- }
- }
- }
-
- private boolean isTypeLike(Type type, Type typeOther) {
- if (type.isStringType()) {
- return typeOther.isStringType();
- } else {
- return type.equals(typeOther);
- }
- }
-
private void executeWithRetry(Set<String> execPartitionNames, Map<TableIf, String> tableWithPartKey)
throws Exception {
int retryCount = 0;
@@ -336,7 +303,7 @@ public class MTMVTask extends AbstractTask {
retryCount++;
LOG.warn("Retrying execution due to exception: {}. Attempt {}/{}, "
- + "taskId {} execPartitionNames {} lastQueryId {}, randomMillis {}",
+ + "taskId {} execPartitionNames {} lastQueryId {}, randomMillis {}",
e.getMessage(), retryCount, retryTime, getTaskId(),
execPartitionNames, lastQueryId, randomMillis);
if (retryCount >= retryTime) {
@@ -361,7 +328,7 @@ public class MTMVTask extends AbstractTask {
// if SELF_MANAGE mv, only have default partition, will not have partitionItem, so we give empty set
UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand
.from(mtmv, mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE
- ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey);
+ ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey, statementContext);
try {
executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, ctx.getStatementContext()));
ctx.setExecutor(executor);
@@ -482,7 +449,7 @@ public class MTMVTask extends AbstractTask {
* @throws DdlException
*/
private void beforeMTMVRefresh() throws AnalysisException, DdlException {
- for (BaseTableInfo tableInfo : relation.getBaseTablesOneLevel()) {
+ for (BaseTableInfo tableInfo : relation.getBaseTablesOneLevelAndFromView()) {
TableIf tableIf = MTMVUtil.getTable(tableInfo);
if (tableIf instanceof MTMVBaseTableIf) {
MTMVBaseTableIf baseTableIf = (MTMVBaseTableIf) tableIf;
@@ -645,7 +612,7 @@ public class MTMVTask extends AbstractTask {
// check if data is fresh
// We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation()
// to avoid rebuilding the baseTable and causing a change in the tableId
- boolean fresh = MTMVPartitionUtil.isMTMVSync(context, relation.getBaseTablesOneLevel(),
+ boolean fresh = MTMVPartitionUtil.isMTMVSync(context, relation.getBaseTablesOneLevelAndFromView(),
mtmv.getExcludedTriggerTables());
if (fresh) {
return Lists.newArrayList();
@@ -656,13 +623,17 @@ public class MTMVTask extends AbstractTask {
}
// We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation()
// to avoid rebuilding the baseTable and causing a change in the tableId
- return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(context, relation.getBaseTablesOneLevel());
+ return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(context, relation.getBaseTablesOneLevelAndFromView());
}
public MTMVTaskContext getTaskContext() {
return taskContext;
}
+ public long getMtmvSchemaChangeVersion() {
+ return mtmvSchemaChangeVersion;
+ }
+
@Override
public String toString() {
return "MTMVTask{"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java
similarity index 50%
copy from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java
copy to fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java
index 5a99d865558..f9ac4f18c1b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java
@@ -15,25 +15,33 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.nereids.rules.analysis;
+package org.apache.doris.mtmv;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.catalog.View;
-import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
import java.util.List;
-/**
- * collect one level relations
- */
-public class CollectOneLevelRelation extends CollectRelation {
- public CollectOneLevelRelation() {}
+public class MTMVAnalyzeQueryInfo {
+ private MTMVRelation relation;
+ private MTMVPartitionInfo mvPartitionInfo;
+ private List<ColumnDefinition> columnDefinitions;
- protected void collectMTMVCandidates(TableIf table, CascadesContext cascadesContext) {
- //do nothing
+ public MTMVAnalyzeQueryInfo(List<ColumnDefinition> columnDefinitions, MTMVPartitionInfo mvPartitionInfo,
+ MTMVRelation relation) {
+ this.columnDefinitions = columnDefinitions;
+ this.mvPartitionInfo = mvPartitionInfo;
+ this.relation = relation;
}
- protected void parseAndCollectFromView(List<String> tableQualifier, View view, CascadesContext parentContext) {
- //do nothing
+ public List<ColumnDefinition> getColumnDefinitions() {
+ return columnDefinitions;
+ }
+
+ public MTMVPartitionInfo getMvPartitionInfo() {
+ return mvPartitionInfo;
+ }
+
+ public MTMVRelation getRelation() {
+ return relation;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java
index b2ef26db1ed..1885d10ae45 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java
@@ -111,4 +111,18 @@ public interface MTMVHookService {
* @param info
*/
void cancelMTMVTask(CancelMTMVTaskInfo info) throws DdlException, MetaNotFoundException, JobException;
+
+ /**
+ * Triggered when baseView is altered
+ *
+ * @param baseViewInfo
+ */
+ void alterView(BaseTableInfo baseViewInfo);
+
+ /**
+ * Triggered when baseView is dropped
+ *
+ * @param baseViewInfo
+ */
+ void dropView(BaseTableInfo baseViewInfo);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
index e449b846414..1edf590f310 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
@@ -216,4 +216,13 @@ public class MTMVJobManager implements MTMVHookService {
return (MTMVJob) Env.getCurrentEnv().getJobManager().getJob(mtmv.getId());
}
+ @Override
+ public void alterView(BaseTableInfo baseViewInfo) {
+
+ }
+
+ @Override
+ public void dropView(BaseTableInfo baseViewInfo) {
+
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
index d34580f6608..9247b716497 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
@@ -26,6 +26,7 @@ import org.apache.doris.datasource.mvcc.MvccUtil;
import com.google.gson.annotations.SerializedName;
import java.util.List;
+import java.util.Objects;
/**
* MTMVPartitionInfo
@@ -129,6 +130,22 @@ public class MTMVPartitionInfo {
partitionColumns));
}
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MTMVPartitionInfo that = (MTMVPartitionInfo) o;
+ return partitionType == that.partitionType && Objects.equals(relatedTable, that.relatedTable)
+ && Objects.equals(relatedCol, that.relatedCol) && Objects.equals(partitionCol,
+ that.partitionCol) && Objects.equals(expr, that.expr);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(partitionType, relatedTable, relatedCol, partitionCol, expr);
+ }
+
// toString() is not easy to find where to call the method
public String toInfoString() {
return "MTMVPartitionInfo{"
@@ -136,6 +153,7 @@ public class MTMVPartitionInfo {
+ ", relatedTable=" + relatedTable
+ ", relatedCol='" + relatedCol + '\''
+ ", partitionCol='" + partitionCol + '\''
+ + ", expr='" + expr + '\''
+ '}';
}
@@ -150,6 +168,7 @@ public class MTMVPartitionInfo {
+ ", relatedTable=" + relatedTable.getTableName()
+ ", relatedCol='" + relatedCol + '\''
+ ", partitionCol='" + partitionCol + '\''
+ + ", expr='" + expr + '\''
+ '}';
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 74cf6396eab..590cbf4951a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -198,7 +198,7 @@ public class MTMVPartitionUtil {
return false;
}
try {
- return isMTMVSync(MTMVRefreshContext.buildContext(mtmv), mtmvRelation.getBaseTablesOneLevel(),
+ return isMTMVSync(MTMVRefreshContext.buildContext(mtmv), mtmvRelation.getBaseTablesOneLevelAndFromView(),
Sets.newHashSet());
} catch (AnalysisException e) {
LOG.warn("isMTMVSync failed: ", e);
@@ -253,7 +253,7 @@ public class MTMVPartitionUtil {
MTMV mtmv = context.getMtmv();
Set<String> relatedPartitionNames = context.getPartitionMappings().get(partitionName);
List<String> res = Lists.newArrayList();
- for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTablesOneLevel()) {
+ for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTablesOneLevelAndFromView()) {
TableIf table = MTMVUtil.getTable(baseTableInfo);
if (!(table instanceof MTMVRelatedTableIf)) {
continue;
@@ -610,11 +610,11 @@ public class MTMVPartitionUtil {
private static Map<Long, Long> getTableVersions(MTMV mtmv) {
Map<Long, Long> res = Maps.newHashMap();
MTMVRelation relation = mtmv.getRelation();
- if (relation == null || relation.getBaseTablesOneLevel() == null) {
+ if (relation == null || relation.getBaseTablesOneLevelAndFromView() == null) {
return res;
}
List<OlapTable> olapTables = Lists.newArrayList();
- for (BaseTableInfo baseTableInfo : relation.getBaseTablesOneLevel()) {
+ for (BaseTableInfo baseTableInfo : relation.getBaseTablesOneLevelAndFromView()) {
TableIf table = null;
try {
table = MTMVUtil.getTable(baseTableInfo);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
index c734e7497f9..ac49d57194e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
@@ -17,32 +17,53 @@
package org.apache.doris.mtmv;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.AggregateType;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.DistributionInfo;
+import org.apache.doris.catalog.DistributionInfo.DistributionInfoType;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.UserException;
import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.job.exception.JobException;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.analyzer.UnboundResultSink;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.exceptions.ParseException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
+import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
+import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo;
+import org.apache.doris.nereids.trees.plans.commands.info.DistributionDescriptor;
+import org.apache.doris.nereids.trees.plans.commands.info.MTMVPartitionDefinition;
import org.apache.doris.nereids.trees.plans.commands.info.SimpleColumnDefinition;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.types.AggStateType;
import org.apache.doris.nereids.types.CharType;
import org.apache.doris.nereids.types.DataType;
@@ -54,6 +75,7 @@ import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.nereids.types.coercion.CharacterType;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -150,27 +172,38 @@ public class MTMVPlanUtil {
ctx.setDatabase(databaseIf.get().getFullName());
}
- public static MTMVRelation generateMTMVRelation(Set<TableIf> tablesInPlan, ConnectContext ctx) {
+ public static MTMVRelation generateMTMVRelation(Set<TableIf> tablesInPlan, Set<TableIf> oneLevelTablesInPlan) {
Set<BaseTableInfo> oneLevelTables = Sets.newHashSet();
Set<BaseTableInfo> allLevelTables = Sets.newHashSet();
Set<BaseTableInfo> oneLevelViews = Sets.newHashSet();
+ Set<BaseTableInfo> allLevelViews = Sets.newHashSet();
+ Set<BaseTableInfo> oneLevelTablesAndFromView = Sets.newHashSet();
for (TableIf table : tablesInPlan) {
BaseTableInfo baseTableInfo = new BaseTableInfo(table);
if (table.getType() == TableType.VIEW) {
- // TODO reopen it after we support mv on view
- // oneLevelViews.add(baseTableInfo);
+ allLevelViews.add(baseTableInfo);
} else {
- oneLevelTables.add(baseTableInfo);
+ oneLevelTablesAndFromView.add(baseTableInfo);
allLevelTables.add(baseTableInfo);
if (table instanceof MTMV) {
allLevelTables.addAll(((MTMV) table).getRelation().getBaseTables());
}
}
}
- return new MTMVRelation(allLevelTables, oneLevelTables, oneLevelViews);
+ for (TableIf table : oneLevelTablesInPlan) {
+ BaseTableInfo baseTableInfo = new BaseTableInfo(table);
+ if (table.getType() == TableType.VIEW) {
+ oneLevelViews.add(baseTableInfo);
+ } else {
+ oneLevelTables.add(baseTableInfo);
+ }
+ }
+ return new MTMVRelation(allLevelTables, oneLevelTables, oneLevelTablesAndFromView, allLevelViews,
+ oneLevelViews);
}
- public static Set<TableIf> getBaseTableFromQuery(String querySql, ConnectContext ctx) {
+ // return allLevelTables:oneLevelTables
+ public static Pair<Set<TableIf>, Set<TableIf>> getBaseTableFromQuery(String querySql, ConnectContext ctx) {
List<StatementBase> statements;
try {
statements = new NereidsParser().parseSQL(querySql);
@@ -185,7 +218,8 @@ public class MTMVPlanUtil {
try {
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN);
- return Sets.newHashSet(ctx.getStatementContext().getTables().values());
+ return Pair.of(Sets.newHashSet(ctx.getStatementContext().getTables().values()),
+ Sets.newHashSet(ctx.getStatementContext().getOneLevelTables().values()));
} finally {
ctx.setStatementContext(original);
}
@@ -296,6 +330,7 @@ public class MTMVPlanUtil {
/**
* generate DataType by Slot
+ *
* @param s
* @param i
* @param ctx
@@ -340,4 +375,216 @@ public class MTMVPlanUtil {
}
return dataType;
}
+
+ public static MTMVAnalyzeQueryInfo analyzeQueryWithSql(MTMV mtmv, ConnectContext ctx) throws UserException {
+ String querySql = mtmv.getQuerySql();
+ MTMVPartitionInfo mvPartitionInfo = mtmv.getMvPartitionInfo();
+ MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition();
+ mtmvPartitionDefinition.setPartitionCol(mvPartitionInfo.getPartitionCol());
+ mtmvPartitionDefinition.setPartitionType(mvPartitionInfo.getPartitionType());
+ Expr expr = mvPartitionInfo.getExpr();
+ if (expr != null) {
+ mtmvPartitionDefinition.setFunctionCallExpression(new NereidsParser().parseExpression(expr.toSql()));
+ }
+ List<String> keys = mtmv.getBaseSchema().stream()
+ .filter(Column::isKey)
+ .map(Column::getName)
+ .collect(Collectors.toList());
+ List<StatementBase> statements;
+ try {
+ statements = new NereidsParser().parseSQL(querySql);
+ } catch (Exception e) {
+ throw new ParseException("Nereids parse failed. " + e.getMessage());
+ }
+ StatementBase parsedStmt = statements.get(0);
+ LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+ DistributionInfo defaultDistributionInfo = mtmv.getDefaultDistributionInfo();
+ DistributionDescriptor distribution = new DistributionDescriptor(defaultDistributionInfo.getType().equals(
+ DistributionInfoType.HASH), defaultDistributionInfo.getAutoBucket(),
+ defaultDistributionInfo.getBucketNum(), Lists.newArrayList(mtmv.getDistributionColumnNames()));
+ return analyzeQuery(ctx, mtmv.getMvProperties(), querySql, mtmvPartitionDefinition, distribution, null,
+ mtmv.getTableProperty().getProperties(), keys, logicalPlan);
+ }
+
+ public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map<String, String> mvProperties,
+ String querySql,
+ MTMVPartitionDefinition mvPartitionDefinition, DistributionDescriptor distribution,
+ List<SimpleColumnDefinition> simpleColumnDefinitions, Map<String, String> properties, List<String> keys,
+ LogicalPlan
+ logicalQuery) throws UserException {
+ try (StatementContext statementContext = ctx.getStatementContext()) {
+ NereidsPlanner planner = new NereidsPlanner(statementContext);
+ // this is for expression column name infer when not use alias
+ LogicalSink<Plan> logicalSink = new UnboundResultSink<>(logicalQuery);
+ // Should not make table without data to empty relation when analyze the related table,
+ // so add disable rules
+ Set<String> tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames();
+ ctx.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES);
+ statementContext.invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
+ Plan plan;
+ try {
+ // must disable constant folding by be, because be constant folding may return wrong type
+ ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_FOLD_CONSTANT_BY_BE, "false");
+ plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
+ } finally {
+ // after operate, roll back the disable rules
+ ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules));
+ statementContext.invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
+ }
+ // can not contain Random function
+ analyzeExpressions(planner.getAnalyzedPlan(), mvProperties);
+ // can not contain partition or tablets
+ boolean containTableQueryOperator = MaterializedViewUtils.containTableQueryOperator(
+ planner.getAnalyzedPlan());
+ if (containTableQueryOperator) {
+ throw new AnalysisException("can not contain invalid expression");
+ }
+
+ Set<TableIf> baseTables = Sets.newHashSet(statementContext.getTables().values());
+ Set<TableIf> oneLevelTables = Sets.newHashSet(statementContext.getOneLevelTables().values());
+ for (TableIf table : baseTables) {
+ if (table.isTemporary()) {
+ throw new AnalysisException("do not support create materialized view on temporary table ("
+ + Util.getTempTableDisplayName(table.getName()) + ")");
+ }
+ }
+ MTMVRelation relation = generateMTMVRelation(baseTables, oneLevelTables);
+ MTMVPartitionInfo mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner);
+ List<ColumnDefinition> columns = MTMVPlanUtil.generateColumns(plan, ctx, mvPartitionInfo.getPartitionCol(),
+ (distribution == null || CollectionUtils.isEmpty(distribution.getCols())) ? Sets.newHashSet()
+ : Sets.newHashSet(distribution.getCols()),
+ simpleColumnDefinitions, properties);
+ analyzeKeys(keys, properties, columns);
+ // analyze column
+ final boolean finalEnableMergeOnWrite = false;
+ Set<String> keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
+ keysSet.addAll(keys);
+ validateColumns(columns, keysSet, finalEnableMergeOnWrite);
+ return new MTMVAnalyzeQueryInfo(columns, mvPartitionInfo, relation);
+ }
+ }
+
+ /**
+ * validate column name
+ */
+ private static void validateColumns(List<ColumnDefinition> columns, Set<String> keysSet,
+ boolean finalEnableMergeOnWrite) throws UserException {
+ Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
+ for (ColumnDefinition col : columns) {
+ if (!colSets.add(col.getName())) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col.getName());
+ }
+ if (col.getType().isVarBinaryType()) {
+ throw new AnalysisException("MTMV do not support varbinary type : " + col.getName());
+ }
+ col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, KeysType.DUP_KEYS);
+ }
+ }
+
+ private static void analyzeKeys(List<String> keys, Map<String, String> properties, List<ColumnDefinition> columns) {
+ boolean enableDuplicateWithoutKeysByDefault = false;
+ try {
+ if (properties != null) {
+ enableDuplicateWithoutKeysByDefault =
+ PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties);
+ }
+ } catch (Exception e) {
+ throw new AnalysisException(e.getMessage(), e.getCause());
+ }
+ if (keys.isEmpty() && !enableDuplicateWithoutKeysByDefault) {
+ keys = Lists.newArrayList();
+ int keyLength = 0;
+ for (ColumnDefinition column : columns) {
+ DataType type = column.getType();
+ Type catalogType = column.getType().toCatalogDataType();
+ keyLength += catalogType.getIndexSize();
+ if (keys.size() >= FeConstants.shortkey_max_column_count
+ || keyLength > FeConstants.shortkey_maxsize_bytes) {
+ if (keys.isEmpty() && type.isStringLikeType()) {
+ keys.add(column.getName());
+ column.setIsKey(true);
+ }
+ break;
+ }
+ if (column.getAggType() != null) {
+ break;
+ }
+ if (!catalogType.couldBeShortKey()) {
+ break;
+ }
+ keys.add(column.getName());
+ column.setIsKey(true);
+ if (type.isVarcharType()) {
+ break;
+ }
+ }
+ }
+ }
+
+ private static void analyzeExpressions(Plan plan, Map<String, String> mvProperties) {
+ boolean enableNondeterministicFunction = Boolean.parseBoolean(
+ mvProperties.get(PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION));
+ if (enableNondeterministicFunction) {
+ return;
+ }
+ List<Expression> functionCollectResult = MaterializedViewUtils.extractNondeterministicFunction(plan);
+ if (!CollectionUtils.isEmpty(functionCollectResult)) {
+ throw new AnalysisException(String.format(
+ "can not contain nonDeterministic expression, the expression is %s. "
+ + "Should add 'enable_nondeterministic_function' = 'true' property "
+ + "when create materialized view if you know the property real meaning entirely",
+ functionCollectResult.stream().map(Expression::toString).collect(Collectors.joining(","))));
+ }
+ }
+
+ public static void ensureMTMVQueryUsable(MTMV mtmv, ConnectContext ctx) throws JobException {
+ MTMVAnalyzeQueryInfo mtmvAnalyzedQueryInfo;
+ try {
+ mtmvAnalyzedQueryInfo = MTMVPlanUtil.analyzeQueryWithSql(mtmv, ctx);
+ } catch (Exception e) {
+ throw new JobException(e.getMessage(), e);
+ }
+ checkColumnIfChange(mtmv, mtmvAnalyzedQueryInfo.getColumnDefinitions());
+ checkMTMVPartitionInfo(mtmv, mtmvAnalyzedQueryInfo.getMvPartitionInfo());
+ }
+
+ private static void checkMTMVPartitionInfo(MTMV mtmv, MTMVPartitionInfo analyzedMvPartitionInfo)
+ throws JobException {
+ MTMVPartitionInfo originalMvPartitionInfo = mtmv.getMvPartitionInfo();
+ if (!analyzedMvPartitionInfo.equals(originalMvPartitionInfo)) {
+ throw new JobException("async materialized view partition info changed, analyzed: %s, original: %s",
+ analyzedMvPartitionInfo.toInfoString(), originalMvPartitionInfo.toInfoString());
+ }
+ }
+
+ private static void checkColumnIfChange(MTMV mtmv, List<ColumnDefinition> analyzedColumnDefinitions)
+ throws JobException {
+ List<Column> analyzedColumns = analyzedColumnDefinitions.stream()
+ .map(ColumnDefinition::translateToCatalogStyle)
+ .collect(Collectors.toList());
+ List<Column> originalColumns = mtmv.getBaseSchema(true);
+ if (analyzedColumns.size() != originalColumns.size()) {
+ throw new JobException(String.format(
+ "column length not equals, please check whether columns of base table have changed, "
+ + "original length is: %s, current length is: %s",
+ originalColumns.size(), analyzedColumns.size()));
+ }
+ for (int i = 0; i < originalColumns.size(); i++) {
+ if (!isTypeLike(originalColumns.get(i).getType(), analyzedColumns.get(i).getType())) {
+ throw new JobException(String.format(
+ "column type not same, please check whether columns of base table have changed, "
+ + "column name is: %s, original type is: %s, current type is: %s",
+ originalColumns.get(i).getName(), originalColumns.get(i).getType().toSql(),
+ analyzedColumns.get(i).getType().toSql()));
+ }
+ }
+ }
+
+ private static boolean isTypeLike(Type type, Type typeOther) {
+ if (type.isStringType()) {
+ return typeOther.isStringType();
+ } else {
+ return type.equals(typeOther);
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
index 490fc8ca2fe..cae7d26c778 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java
@@ -152,19 +152,19 @@ public class MTMVRefreshPartitionSnapshot {
return;
}
MTMVRelation relation = mtmv.getRelation();
- if (relation == null || CollectionUtils.isEmpty(relation.getBaseTablesOneLevel())) {
+ if (relation == null || CollectionUtils.isEmpty(relation.getBaseTablesOneLevelAndFromView())) {
return;
}
for (Entry<Long, MTMVSnapshotIf> entry : tables.entrySet()) {
Optional<BaseTableInfo> tableInfo = getByTableId(entry.getKey(),
- relation.getBaseTablesOneLevel());
+ relation.getBaseTablesOneLevelAndFromView());
if (tableInfo.isPresent()) {
tablesInfo.put(tableInfo.get(), entry.getValue());
} else {
String msg = String.format(
"Failed to get table info based on id during compatibility process, "
+ "tableId: %s, relationTables: %s",
- entry.getKey(), relation.getBaseTablesOneLevel());
+ entry.getKey(), relation.getBaseTablesOneLevelAndFromView());
LOG.warn(msg);
throw new Exception(msg);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
index 148d2d00884..7e784932e7b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java
@@ -25,22 +25,36 @@ import org.apache.commons.collections.CollectionUtils;
import java.util.Set;
public class MTMVRelation {
- // if mtmv query sql is `select * from view1`;
- // and `view1` query sql is `select * from table1 join table2`
- // then baseTables will include: `table1` and `table2`
- // baseViews will include `view1`
+ // t1 => v1 => v2
+ // t2 => mv1
+ // mv1 join v2 => mv2
+ //
+ // data of mv2 is:
+ //
+ // baseTables => mv1,t1,t2
+ // baseTablesOneLevelAndFromView => mv1,t1
+ // baseTablesOneLevel => mv1
+ // baseViews => v2,v1
+ // baseViewsOneLevel => v2
@SerializedName("bt")
private Set<BaseTableInfo> baseTables;
@SerializedName("bv")
private Set<BaseTableInfo> baseViews;
@SerializedName("btol")
private Set<BaseTableInfo> baseTablesOneLevel;
+ @SerializedName("btolafv")
+ private Set<BaseTableInfo> baseTablesOneLevelAndFromView;
+ @SerializedName("bvol")
+ private Set<BaseTableInfo> baseViewsOneLevel;
public MTMVRelation(Set<BaseTableInfo> baseTables, Set<BaseTableInfo> baseTablesOneLevel,
- Set<BaseTableInfo> baseViews) {
+ Set<BaseTableInfo> baseTablesOneLevelAndFromView, Set<BaseTableInfo> baseViews,
+ Set<BaseTableInfo> baseViewsOneLevel) {
this.baseTables = baseTables;
this.baseTablesOneLevel = baseTablesOneLevel;
+ this.baseTablesOneLevelAndFromView = baseTablesOneLevelAndFromView;
this.baseViews = baseViews;
+ this.baseViewsOneLevel = baseViewsOneLevel;
}
public Set<BaseTableInfo> getBaseTables() {
@@ -52,6 +66,16 @@ public class MTMVRelation {
return baseTablesOneLevel == null ? baseTables : baseTablesOneLevel;
}
+ public Set<BaseTableInfo> getBaseTablesOneLevelAndFromView() {
+ // For compatibility, previously created MTMV may not have baseTablesOneLevelAndFromView
+ return CollectionUtils.isEmpty(baseTablesOneLevelAndFromView) ? baseTablesOneLevel
+ : baseTablesOneLevelAndFromView;
+ }
+
+ public Set<BaseTableInfo> getBaseViewsOneLevel() {
+ return baseViewsOneLevel;
+ }
+
public Set<BaseTableInfo> getBaseViews() {
return baseViews;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
index d8a2146210b..2cb4f2ffd35 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
@@ -59,19 +59,28 @@ import java.util.function.BiPredicate;
public class MTMVRelationManager implements MTMVHookService {
private static final Logger LOG = LogManager.getLogger(MTMVRelationManager.class);
// when
- // create mv1 as select * from table1;
+ // create v1 as select * from table1
+ // create v2 as select * from v1
+ // create mv1 as select * from v1;
// create mv2 as select * from mv1;
- // `tableMTMVs` will have 3 pair: table1 ==> mv1,mv1==>mv2, table1 ==> mv2
- // `tableMTMVsOneLevel` will have 2 pair: table1 ==> mv1,mv1==>mv2
+ // `tableMTMVs` will have 3 pair: table1 ==> mv1, mv1==>mv2, table1 ==> mv2
+ // `tableMTMVsOneLevelAndFromView` will have 2 pair: table1 ==> mv1, mv1==>mv2
+ // `viewMTMVs` will have 2 pair: v1 ==> mv1, v2 ==> mv1
private final Map<BaseTableInfo, Set<BaseTableInfo>> tableMTMVs = Maps.newConcurrentMap();
- private final Map<BaseTableInfo, Set<BaseTableInfo>> tableMTMVsOneLevel = Maps.newConcurrentMap();
+ private final Map<BaseTableInfo, Set<BaseTableInfo>> tableMTMVsOneLevelAndFromView = Maps.newConcurrentMap();
+ // view => mtmv
+ private final Map<BaseTableInfo, Set<BaseTableInfo>> viewMTMVs = Maps.newConcurrentMap();
public Set<BaseTableInfo> getMtmvsByBaseTable(BaseTableInfo table) {
return tableMTMVs.getOrDefault(table, ImmutableSet.of());
}
- public Set<BaseTableInfo> getMtmvsByBaseTableOneLevel(BaseTableInfo table) {
- return tableMTMVsOneLevel.getOrDefault(table, ImmutableSet.of());
+ public Set<BaseTableInfo> getMtmvsByBaseView(BaseTableInfo table) {
+ return viewMTMVs.getOrDefault(table, ImmutableSet.of());
+ }
+
+ public Set<BaseTableInfo> getMtmvsByBaseTableOneLevelAndFromView(BaseTableInfo table) {
+ return tableMTMVsOneLevelAndFromView.getOrDefault(table, ImmutableSet.of());
}
/**
@@ -149,11 +158,18 @@ public class MTMVRelationManager implements MTMVHookService {
return tableMTMVs.get(baseTableInfo);
}
- private Set<BaseTableInfo> getOrCreateMTMVsOneLevel(BaseTableInfo baseTableInfo) {
- if (!tableMTMVsOneLevel.containsKey(baseTableInfo)) {
- tableMTMVsOneLevel.put(baseTableInfo, Sets.newConcurrentHashSet());
+ private Set<BaseTableInfo> getOrCreateMTMVsView(BaseTableInfo baseTableInfo) {
+ if (!viewMTMVs.containsKey(baseTableInfo)) {
+ viewMTMVs.put(baseTableInfo, Sets.newConcurrentHashSet());
+ }
+ return viewMTMVs.get(baseTableInfo);
+ }
+
+ private Set<BaseTableInfo> getOrCreateMTMVsOneLevelAndFromView(BaseTableInfo baseTableInfo) {
+ if (!tableMTMVsOneLevelAndFromView.containsKey(baseTableInfo)) {
+ tableMTMVsOneLevelAndFromView.put(baseTableInfo, Sets.newConcurrentHashSet());
}
- return tableMTMVsOneLevel.get(baseTableInfo);
+ return tableMTMVsOneLevelAndFromView.get(baseTableInfo);
}
public void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) {
@@ -167,8 +183,8 @@ public class MTMVRelationManager implements MTMVHookService {
return;
}
addMTMVTables(relation.getBaseTables(), mtmvInfo);
- addMTMVTables(relation.getBaseViews(), mtmvInfo);
- addMTMVTablesOneLevel(relation.getBaseTablesOneLevel(), mtmvInfo);
+ addMTMVViews(relation.getBaseViews(), mtmvInfo);
+ addMTMVTablesOneLevelAndFromView(relation.getBaseTablesOneLevelAndFromView(), mtmvInfo);
}
private void addMTMVTables(Set<BaseTableInfo> baseTables, BaseTableInfo mtmvInfo) {
@@ -180,12 +196,21 @@ public class MTMVRelationManager implements MTMVHookService {
}
}
- private void addMTMVTablesOneLevel(Set<BaseTableInfo> baseTables, BaseTableInfo mtmvInfo) {
+ private void addMTMVViews(Set<BaseTableInfo> baseTables, BaseTableInfo mtmvInfo) {
+ if (CollectionUtils.isEmpty(baseTables)) {
+ return;
+ }
+ for (BaseTableInfo baseTableInfo : baseTables) {
+ getOrCreateMTMVsView(baseTableInfo).add(mtmvInfo);
+ }
+ }
+
+ private void addMTMVTablesOneLevelAndFromView(Set<BaseTableInfo> baseTables, BaseTableInfo mtmvInfo) {
if (CollectionUtils.isEmpty(baseTables)) {
return;
}
for (BaseTableInfo baseTableInfo : baseTables) {
- getOrCreateMTMVsOneLevel(baseTableInfo).add(mtmvInfo);
+ getOrCreateMTMVsOneLevelAndFromView(baseTableInfo).add(mtmvInfo);
}
}
@@ -193,7 +218,10 @@ public class MTMVRelationManager implements MTMVHookService {
for (Set<BaseTableInfo> sets : tableMTMVs.values()) {
sets.remove(mtmvInfo);
}
- for (Set<BaseTableInfo> sets : tableMTMVsOneLevel.values()) {
+ for (Set<BaseTableInfo> sets : viewMTMVs.values()) {
+ sets.remove(mtmvInfo);
+ }
+ for (Set<BaseTableInfo> sets : tableMTMVsOneLevelAndFromView.values()) {
sets.remove(mtmvInfo);
}
}
@@ -286,8 +314,47 @@ public class MTMVRelationManager implements MTMVHookService {
}
+ /**
+ * update mtmv status to `SCHEMA_CHANGE` and drop snapshot
+ *
+ * @param baseViewInfo
+ */
+ @Override
+ public void alterView(BaseTableInfo baseViewInfo) {
+ processBaseViewChange(baseViewInfo, "The base view has been updated:");
+ }
+
+ /**
+ * update mtmv status to `SCHEMA_CHANGE` and drop snapshot
+ *
+ * @param baseViewInfo
+ */
+ @Override
+ public void dropView(BaseTableInfo baseViewInfo) {
+ processBaseViewChange(baseViewInfo, "The base view has been dropped:");
+ }
+
+ private void processBaseViewChange(BaseTableInfo baseViewInfo, String msgPrefix) {
+ Set<BaseTableInfo> mtmvsByBaseView = getMtmvsByBaseView(baseViewInfo);
+ LOG.info("processBaseViewChange, baseViewInfo: {}, mtmvsByBaseView: {}", baseViewInfo, mtmvsByBaseView);
+ if (CollectionUtils.isEmpty(mtmvsByBaseView)) {
+ return;
+ }
+ for (BaseTableInfo mtmvInfo : mtmvsByBaseView) {
+ MTMV mtmv = null;
+ try {
+ mtmv = MTMVUtil.getMTMV(mtmvInfo);
+ } catch (AnalysisException e) {
+ LOG.warn(e);
+ continue;
+ }
+ String schemaChangeDetail = msgPrefix + baseViewInfo;
+ mtmv.processBaseViewChange(schemaChangeDetail);
+ }
+ }
+
private void processBaseTableChange(BaseTableInfo baseTableInfo, String msgPrefix) {
- Set<BaseTableInfo> mtmvsByBaseTable = getMtmvsByBaseTableOneLevel(baseTableInfo);
+ Set<BaseTableInfo> mtmvsByBaseTable = getMtmvsByBaseTableOneLevelAndFromView(baseTableInfo);
if (CollectionUtils.isEmpty(mtmvsByBaseTable)) {
return;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
index 751362be92d..c95e1262281 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
@@ -91,7 +91,7 @@ public class MTMVRewriteUtil {
}
try {
if (MTMVPartitionUtil.isMTMVPartitionSync(refreshContext, partition.getName(),
- mtmvRelation.getBaseTablesOneLevel(),
+ mtmvRelation.getBaseTablesOneLevelAndFromView(),
forceConsistent ? ImmutableSet.of() : mtmv.getQueryRewriteConsistencyRelaxedTables())) {
res.add(partition);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java
index 96e7226a20e..5053b2a0467 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java
@@ -122,6 +122,22 @@ public class MTMVService implements EventListener {
}
}
+ public void dropView(BaseTableInfo baseViewInfo) {
+ Objects.requireNonNull(baseViewInfo, "baseViewInfo can not be null");
+ LOG.info("dropView, view: {}", baseViewInfo);
+ for (MTMVHookService mtmvHookService : hooks.values()) {
+ mtmvHookService.dropView(baseViewInfo);
+ }
+ }
+
+ public void alterView(BaseTableInfo baseViewInfo) {
+ Objects.requireNonNull(baseViewInfo, "baseViewInfo can not be null");
+ LOG.info("alterView, view: {}", baseViewInfo);
+ for (MTMVHookService mtmvHookService : hooks.values()) {
+ mtmvHookService.alterView(baseViewInfo);
+ }
+ }
+
public void refreshComplete(MTMV mtmv, MTMVRelation cache, MTMVTask task) {
Objects.requireNonNull(mtmv, "mtmv can not be null");
Objects.requireNonNull(task, "task can not be null");
@@ -180,7 +196,7 @@ public class MTMVService implements EventListener {
} catch (AnalysisException e) {
throw new EventException(e);
}
- Set<BaseTableInfo> mtmvs = relationManager.getMtmvsByBaseTableOneLevel(
+ Set<BaseTableInfo> mtmvs = relationManager.getMtmvsByBaseTableOneLevelAndFromView(
new BaseTableInfo(table));
for (BaseTableInfo baseTableInfo : mtmvs) {
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java
index fa0a948f3de..01a08f823c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java
@@ -85,6 +85,14 @@ public class MTMVUtil {
return (MTMVRelatedTableIf) relatedTable;
}
+ public static MTMV getMTMV(BaseTableInfo baseTableInfo) throws AnalysisException {
+ TableIf table = getTable(baseTableInfo);
+ if (!(table instanceof MTMV)) {
+ throw new AnalysisException(String.format("table is not MTMV, table: %s", baseTableInfo));
+ }
+ return (MTMV) table;
+ }
+
public static MTMV getMTMV(long dbId, long mtmvId) throws DdlException, MetaNotFoundException {
Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId);
return (MTMV) db.getTableOrMetaException(mtmvId, TableType.MATERIALIZED_VIEW);
@@ -97,7 +105,7 @@ public class MTMVUtil {
* @return
*/
public static boolean mtmvContainsExternalTable(MTMV mtmv) {
- Set<BaseTableInfo> baseTables = mtmv.getRelation().getBaseTablesOneLevel();
+ Set<BaseTableInfo> baseTables = mtmv.getRelation().getBaseTablesOneLevelAndFromView();
for (BaseTableInfo baseTableInfo : baseTables) {
if (!baseTableInfo.isInternalTable()) {
return true;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index efb53b66822..6980765049f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -272,8 +272,8 @@ public class CascadesContext implements ScheduleContext {
}
}
- public TableCollectAndHookInitializer newTableCollector() {
- return new TableCollectAndHookInitializer(this);
+ public TableCollectAndHookInitializer newTableCollector(boolean firstLevel) {
+ return new TableCollectAndHookInitializer(this, firstLevel);
}
public Analyzer newAnalyzer() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 28e7a477f9d..81d5108ce2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -394,7 +394,7 @@ public class NereidsPlanner extends Planner {
if (LOG.isDebugEnabled()) {
LOG.debug("Start collect and lock table");
}
- keepOrShowPlanProcess(showPlanProcess, () -> cascadesContext.newTableCollector().collect());
+ keepOrShowPlanProcess(showPlanProcess, () -> cascadesContext.newTableCollector(true).collect());
statementContext.lock();
cascadesContext.setCteContext(new CTEContext());
NereidsTracer.logImportantTime("EndCollectAndLockTables");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 746efa3cf95..5829a0c9b56 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -188,6 +188,14 @@ public class StatementContext implements Closeable {
// tables in this query directly
private final Map<List<String>, TableIf> tables = Maps.newHashMap();
+ // onelevel tables in this query directly,
+ // if
+ // create v1 as select * from t1
+ // create v2 as select * from v1
+ // current query is: select * from v2 join t2
+ // oneLevelTables will have two data: v2, t2,
+ // tables will have 4 data: t1, v1, v2, t2
+ private final Map<List<String>, TableIf> oneLevelTables = Maps.newHashMap();
// tables maybe used by mtmv rewritten in this query,
// this contains mvs which use table in tables and the tables in mvs
// such as
@@ -275,6 +283,8 @@ public class StatementContext implements Closeable {
private boolean isInsert = false;
+ private Optional<Map<TableIf, Set<Expression>>> mvRefreshPredicates = Optional.empty();
+
public StatementContext() {
this(ConnectContext.get(), null, 0);
}
@@ -357,6 +367,10 @@ public class StatementContext implements Closeable {
return tables;
}
+ public Map<List<String>, TableIf> getOneLevelTables() {
+ return oneLevelTables;
+ }
+
public Set<MTMV> getCandidateMTMVs() {
return candidateMTMVs;
}
@@ -994,4 +1008,13 @@ public class StatementContext implements Closeable {
public boolean isInsert() {
return isInsert;
}
+
+ public Optional<Map<TableIf, Set<Expression>>> getMvRefreshPredicates() {
+ return mvRefreshPredicates;
+ }
+
+ public void setMvRefreshPredicates(
+ Map<TableIf, Set<Expression>> mvRefreshPredicates) {
+ this.mvRefreshPredicates = Optional.of(mvRefreshPredicates);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TableCollectAndHookInitializer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TableCollectAndHookInitializer.java
index 01ce6687ecb..02b2dc76fb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TableCollectAndHookInitializer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TableCollectAndHookInitializer.java
@@ -33,21 +33,21 @@ import java.util.List;
*/
public class TableCollectAndHookInitializer extends AbstractBatchJobExecutor {
- public static final List<RewriteJob> COLLECT_JOBS = buildCollectTableJobs();
+ public final List<RewriteJob> collectJobs;
/**
* constructor of Analyzer. For view, we only do bind relation since other analyze step will do by outer Analyzer.
*
* @param cascadesContext current context for analyzer
*/
- public TableCollectAndHookInitializer(CascadesContext cascadesContext) {
+ public TableCollectAndHookInitializer(CascadesContext cascadesContext, boolean firstLevel) {
super(cascadesContext);
-
+ collectJobs = buildCollectTableJobs(firstLevel);
}
@Override
public List<RewriteJob> getJobs() {
- return COLLECT_JOBS;
+ return collectJobs;
}
/**
@@ -57,17 +57,17 @@ public class TableCollectAndHookInitializer extends AbstractBatchJobExecutor {
execute();
}
- private static List<RewriteJob> buildCollectTableJobs() {
+ private static List<RewriteJob> buildCollectTableJobs(boolean firstLevel) {
return notTraverseChildrenOf(
ImmutableSet.of(LogicalView.class),
- TableCollectAndHookInitializer::buildCollectorJobs
+ () -> TableCollectAndHookInitializer.buildCollectorJobs(firstLevel)
);
}
- private static List<RewriteJob> buildCollectorJobs() {
+ private static List<RewriteJob> buildCollectorJobs(boolean firstLevel) {
return jobs(
topDown(new AddInitMaterializationHook()),
- topDown(new CollectRelation())
+ topDown(new CollectRelation(firstLevel))
);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java
index 5a99d865558..69be32122f8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java
@@ -27,7 +27,9 @@ import java.util.List;
* collect one level relations
*/
public class CollectOneLevelRelation extends CollectRelation {
- public CollectOneLevelRelation() {}
+ public CollectOneLevelRelation() {
+ super(true);
+ }
protected void collectMTMVCandidates(TableIf table, CascadesContext cascadesContext) {
//do nothing
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
index 53650234610..914220f177e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
@@ -67,7 +67,11 @@ public class CollectRelation implements AnalysisRuleFactory {
private static final Logger LOG = LogManager.getLogger(CollectRelation.class);
- public CollectRelation() {}
+ private boolean firstLevel;
+
+ public CollectRelation(boolean firstLevel) {
+ this.firstLevel = firstLevel;
+ }
@Override
public List<Rule> buildRules() {
@@ -104,7 +108,7 @@ public class CollectRelation implements AnalysisRuleFactory {
LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child();
CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext(
cascadesContext, parsedCtePlan, outerCteCtx);
- innerCascadesCtx.newTableCollector().collect();
+ innerCascadesCtx.newTableCollector(true).collect();
LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan();
// cteId is not used in CollectTable stage
CTEId cteId = new CTEId(0);
@@ -124,7 +128,7 @@ public class CollectRelation implements AnalysisRuleFactory {
CascadesContext subqueryContext = CascadesContext.newContextWithCteContext(
ctx.cascadesContext, subqueryExpr.getQueryPlan(), ctx.cteContext);
subqueryContext.keepOrShowPlanProcess(ctx.cascadesContext.showPlanProcess(),
- () -> subqueryContext.newTableCollector().collect());
+ () -> subqueryContext.newTableCollector(true).collect());
ctx.cascadesContext.addPlanProcesses(subqueryContext.getPlanProcesses());
}
});
@@ -189,8 +193,11 @@ public class CollectRelation implements AnalysisRuleFactory {
if (cascadesContext.getRewritePlan() instanceof UnboundDictionarySink) {
table = ((UnboundDictionarySink) cascadesContext.getRewritePlan()).getDictionary();
} else {
- table = cascadesContext.getConnectContext().getStatementContext()
- .getAndCacheTable(tableQualifier, tableFrom, unboundRelation);
+ StatementContext statementContext = cascadesContext.getConnectContext().getStatementContext();
+ table = statementContext.getAndCacheTable(tableQualifier, tableFrom, unboundRelation);
+ if (firstLevel) {
+ statementContext.getOneLevelTables().put(tableQualifier, table);
+ }
}
if (LOG.isDebugEnabled()) {
LOG.debug("collect table {} from {}", nameParts, tableFrom);
@@ -281,7 +288,7 @@ public class CollectRelation implements AnalysisRuleFactory {
CascadesContext viewContext = CascadesContext.initContext(
parentContext.getStatementContext(), parsedViewPlan, PhysicalProperties.ANY);
viewContext.keepOrShowPlanProcess(parentContext.showPlanProcess(),
- () -> viewContext.newTableCollector().collect());
+ () -> viewContext.newTableCollector(false).collect());
parentContext.addPlanProcesses(viewContext.getPlanProcesses());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index cb62eaf42f1..5c273ebfb01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.constraint.TableIdentifier;
+import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.datasource.mvcc.MvccUtil;
import org.apache.doris.mtmv.BaseTableInfo;
@@ -677,8 +678,14 @@ public class MaterializedViewUtils {
table.getName()));
return null;
}
- Set<Column> partitionColumnSet = new HashSet<>(
- relatedTable.getPartitionColumns(MvccUtil.getSnapshotFromContext(relatedTable)));
+ Set<String> partitionColumnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
+ try {
+ partitionColumnSet.addAll(relatedTable.getPartitionColumnNames(
+ MvccUtil.getSnapshotFromContext(relatedTable)));
+ } catch (DdlException e) {
+ context.addFailReason(e.getMessage());
+ return null;
+ }
Column mvReferenceColumn = contextPartitionColumn.getOriginalColumn().get();
Expr definExpr = mvReferenceColumn.getDefineExpr();
if (definExpr instanceof SlotRef) {
@@ -687,7 +694,7 @@ public class MaterializedViewUtils {
mvReferenceColumn = referenceRollupColumn;
}
}
- if (partitionColumnSet.contains(mvReferenceColumn)
+ if (partitionColumnSet.contains(mvReferenceColumn.getName())
&& (!mvReferenceColumn.isAllowNull() || relatedTable.isPartitionColumnAllowNull())) {
context.addTableColumn(table, mvReferenceColumn);
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
index 27758175527..8411bf14085 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
@@ -32,6 +32,7 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.mvcc.MvccUtil;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVRelatedTableIf;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator;
@@ -101,16 +102,16 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
* @param mv materialize view
* @param partitionNames update partitions in mv and tables
* @param tableWithPartKey the partitions key for different table
+ * @param statementContext statementContext
* @return command
*/
public static UpdateMvByPartitionCommand from(MTMV mv, Set<String> partitionNames,
- Map<TableIf, String> tableWithPartKey) throws UserException {
+ Map<TableIf, String> tableWithPartKey, StatementContext statementContext) throws UserException {
NereidsParser parser = new NereidsParser();
Map<TableIf, Set<Expression>> predicates =
constructTableWithPredicates(mv, partitionNames, tableWithPartKey);
List<String> parts = constructPartsForMv(partitionNames);
Plan plan = parser.parseSingle(mv.getQuerySql());
- plan = plan.accept(new PredicateAdder(), new PredicateAddContext(predicates));
if (plan instanceof Sink) {
plan = plan.child(0);
}
@@ -120,6 +121,7 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
LOG.debug("MTMVTask plan for mvName: {}, partitionNames: {}, plan: {}", mv.getName(), partitionNames,
sink.treeString());
}
+ statementContext.setMvRefreshPredicates(predicates);
return new UpdateMvByPartitionCommand(sink);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index 11947d2da84..ff2f6e4c378 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -24,18 +24,14 @@ import org.apache.doris.analysis.RangePartitionDesc;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PartitionType;
-import org.apache.doris.catalog.TableIf;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.catalog.View;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
-import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DynamicPartitionUtil;
import org.apache.doris.common.util.PropertyAnalyzer;
-import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mtmv.MTMVAnalyzeQueryInfo;
import org.apache.doris.mtmv.MTMVPartitionInfo;
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import org.apache.doris.mtmv.MTMVPartitionUtil;
@@ -47,30 +43,20 @@ import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.CascadesContext;
-import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.UnboundResultSink;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
-import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo.AnalyzerForCreateView;
import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo.PlanSlotFinder;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
-import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
-import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.SessionVariable;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -78,7 +64,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
@@ -155,12 +140,8 @@ public class CreateMTMVInfo extends CreateTableInfo {
throw new AnalysisException(message);
}
analyzeProperties();
- analyzeQuery(ctx, this.mvProperties);
- // analyze column
- final boolean finalEnableMergeOnWrite = false;
- Set<String> keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
- keysSet.addAll(keys);
- validateColumns(this.columns, keysSet, finalEnableMergeOnWrite);
+ analyzeQuery(ctx);
+ this.partitionDesc = generatePartitionDesc(ctx);
if (distribution == null) {
throw new AnalysisException("Create async materialized view should contain distribution desc");
}
@@ -185,21 +166,6 @@ public class CreateMTMVInfo extends CreateTableInfo {
setTableInformation(ctx);
}
- /**validate column name*/
- public void validateColumns(List<ColumnDefinition> columns, Set<String> keysSet,
- boolean finalEnableMergeOnWrite) throws UserException {
- Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
- for (ColumnDefinition col : columns) {
- if (!colSets.add(col.getName())) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col.getName());
- }
- if (col.getType().isVarBinaryType()) {
- throw new AnalysisException("MTMV do not support varbinary type : " + col.getName());
- }
- col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, KeysType.DUP_KEYS);
- }
- }
-
private void rewriteQuerySql(ConnectContext ctx) {
analyzeAndFillRewriteSqlMap(querySql, ctx);
querySql = BaseViewInfo.rewriteSql(ctx.getStatementContext().getIndexInSqlToString(), querySql);
@@ -241,98 +207,13 @@ public class CreateMTMVInfo extends CreateTableInfo {
/**
* analyzeQuery
*/
- public void analyzeQuery(ConnectContext ctx, Map<String, String> mvProperties) {
- try (StatementContext statementContext = ctx.getStatementContext()) {
- NereidsPlanner planner = new NereidsPlanner(statementContext);
- // this is for expression column name infer when not use alias
- LogicalSink<Plan> logicalSink = new UnboundResultSink<>(logicalQuery);
- // Should not make table without data to empty relation when analyze the related table,
- // so add disable rules
- Set<String> tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames();
- ctx.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES);
- statementContext.invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
- Plan plan;
- try {
- // must disable constant folding by be, because be constant folding may return wrong type
- ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_FOLD_CONSTANT_BY_BE, "false");
- plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
- } finally {
- // after operate, roll back the disable rules
- ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules));
- statementContext.invalidCache(SessionVariable.DISABLE_NEREIDS_RULES);
- }
- // can not contain VIEW or MTMV
- analyzeBaseTables(planner.getAnalyzedPlan());
- // can not contain Random function
- analyzeExpressions(planner.getAnalyzedPlan(), mvProperties);
- // can not contain partition or tablets
- boolean containTableQueryOperator = MaterializedViewUtils.containTableQueryOperator(
- planner.getAnalyzedPlan());
- if (containTableQueryOperator) {
- throw new AnalysisException("can not contain invalid expression");
- }
-
- Set<TableIf> baseTables = Sets.newHashSet(statementContext.getTables().values());
- for (TableIf table : baseTables) {
- if (table.isTemporary()) {
- throw new AnalysisException("do not support create materialized view on temporary table ("
- + Util.getTempTableDisplayName(table.getName()) + ")");
- }
- }
- getMTMVRelation(baseTables, ctx);
- this.mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner);
- this.partitionDesc = generatePartitionDesc(ctx);
- columns = MTMVPlanUtil.generateColumns(plan, ctx, mvPartitionInfo.getPartitionCol(),
- (distribution == null || CollectionUtils.isEmpty(distribution.getCols())) ? Sets.newHashSet()
- : Sets.newHashSet(distribution.getCols()),
- simpleColumnDefinitions, properties);
- analyzeKeys();
- }
- }
-
- private void analyzeKeys() {
- boolean enableDuplicateWithoutKeysByDefault = false;
- try {
- if (properties != null) {
- enableDuplicateWithoutKeysByDefault =
- PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties);
- }
- } catch (Exception e) {
- throw new AnalysisException(e.getMessage(), e.getCause());
- }
- if (keys.isEmpty() && !enableDuplicateWithoutKeysByDefault) {
- keys = Lists.newArrayList();
- int keyLength = 0;
- for (ColumnDefinition column : columns) {
- DataType type = column.getType();
- Type catalogType = column.getType().toCatalogDataType();
- keyLength += catalogType.getIndexSize();
- if (keys.size() >= FeConstants.shortkey_max_column_count
- || keyLength > FeConstants.shortkey_maxsize_bytes) {
- if (keys.isEmpty() && type.isStringLikeType()) {
- keys.add(column.getName());
- column.setIsKey(true);
- }
- break;
- }
- if (column.getAggType() != null) {
- break;
- }
- if (!catalogType.couldBeShortKey()) {
- break;
- }
- keys.add(column.getName());
- column.setIsKey(true);
- if (type.isVarcharType()) {
- break;
- }
- }
- }
- }
-
- // Should use analyzed plan for collect views and tables
- private void getMTMVRelation(Set<TableIf> tables, ConnectContext ctx) {
- this.relation = MTMVPlanUtil.generateMTMVRelation(tables, ctx);
+ public void analyzeQuery(ConnectContext ctx) throws UserException {
+ MTMVAnalyzeQueryInfo mtmvAnalyzeQueryInfo = MTMVPlanUtil.analyzeQuery(ctx, this.mvProperties, this.querySql,
+ this.mvPartitionDefinition, this.distribution, this.simpleColumnDefinitions, this.properties, this.keys,
+ this.logicalQuery);
+ this.mvPartitionInfo = mtmvAnalyzeQueryInfo.getMvPartitionInfo();
+ this.columns = mtmvAnalyzeQueryInfo.getColumnDefinitions();
+ this.relation = mtmvAnalyzeQueryInfo.getRelation();
}
private PartitionDesc generatePartitionDesc(ConnectContext ctx) {
@@ -370,41 +251,6 @@ public class CreateMTMVInfo extends CreateTableInfo {
}
}
- private void analyzeBaseTables(Plan plan) {
- List<Object> subQuerys = plan.collectToList(node -> node instanceof LogicalSubQueryAlias);
- for (Object subquery : subQuerys) {
- List<String> qualifier = ((LogicalSubQueryAlias) subquery).getQualifier();
- if (!CollectionUtils.isEmpty(qualifier) && qualifier.size() == 3) {
- try {
- TableIf table = Env.getCurrentEnv().getCatalogMgr()
- .getCatalogOrAnalysisException(qualifier.get(0))
- .getDbOrAnalysisException(qualifier.get(1)).getTableOrAnalysisException(qualifier.get(2));
- if (table instanceof View) {
- throw new AnalysisException("can not contain VIEW");
- }
- } catch (org.apache.doris.common.AnalysisException e) {
- LOG.warn(e.getMessage(), e);
- }
- }
- }
- }
-
- private void analyzeExpressions(Plan plan, Map<String, String> mvProperties) {
- boolean enableNondeterministicFunction = Boolean.parseBoolean(
- mvProperties.get(PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION));
- if (enableNondeterministicFunction) {
- return;
- }
- List<Expression> functionCollectResult = MaterializedViewUtils.extractNondeterministicFunction(plan);
- if (!CollectionUtils.isEmpty(functionCollectResult)) {
- throw new AnalysisException(String.format(
- "can not contain nonDeterministic expression, the expression is %s. "
- + "Should add 'enable_nondeterministic_function' = 'true' property "
- + "when create materialized view if you know the property real meaning entirely",
- functionCollectResult.stream().map(Expression::toString).collect(Collectors.joining(","))));
- }
- }
-
/**
* set CreateTableInfo Information
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java
index 750e36d41a6..9e7e7bc19cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java
@@ -53,7 +53,9 @@ import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
+import java.util.Set;
/**
* Logical Check Policy
@@ -140,6 +142,11 @@ public class LogicalCheckPolicy<CHILD_TYPE extends Plan> extends LogicalUnary<CH
if (!(logicalPlan instanceof CatalogRelation || logicalPlan instanceof LogicalView)) {
return RelatedPolicy.NO_POLICY;
}
+ Optional<Map<TableIf, Set<Expression>>> mvRefreshPredicates = cascadesContext.getStatementContext()
+ .getMvRefreshPredicates();
+ if (mvRefreshPredicates.isPresent()) {
+ return findPolicyByMvRefresh(mvRefreshPredicates.get(), logicalPlan);
+ }
ConnectContext connectContext = cascadesContext.getConnectContext();
AccessControllerManager accessManager = connectContext.getEnv().getAccessManager();
UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
@@ -202,6 +209,16 @@ public class LogicalCheckPolicy<CHILD_TYPE extends Plan> extends LogicalUnary<CH
);
}
+ private RelatedPolicy findPolicyByMvRefresh(Map<TableIf, Set<Expression>> mvRefreshPredicates,
+ LogicalPlan logicalPlan) {
+ TableIf table = logicalPlan instanceof CatalogRelation ? ((CatalogRelation) logicalPlan).getTable()
+ : ((LogicalView<?>) logicalPlan).getView();
+ if (mvRefreshPredicates.containsKey(table)) {
+ return new RelatedPolicy(Optional.of(ExpressionUtils.or(mvRefreshPredicates.get(table))), Optional.empty());
+ }
+ return RelatedPolicy.NO_POLICY;
+ }
+
private Expression mergeRowPolicy(List<? extends RowFilterPolicy> policies) {
List<Expression> orList = new ArrayList<>();
List<Expression> andList = new ArrayList<>();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index c40dd0a03f2..c22147ffdd6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -71,7 +71,6 @@ import org.apache.doris.job.extensions.insert.streaming.StreamingInsertJob;
import org.apache.doris.job.extensions.insert.streaming.StreamingInsertTask;
import org.apache.doris.job.extensions.mtmv.MTMVJob;
import org.apache.doris.job.task.AbstractTask;
-import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVPartitionUtil;
import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVStatus;
@@ -131,8 +130,8 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
public class MetadataGenerator {
private static final Logger LOG = LogManager.getLogger(MetadataGenerator.class);
@@ -712,19 +711,19 @@ public class MetadataGenerator {
continue;
}
MTMVRelation relation = ((MTMV) table).getRelation();
- Set<BaseTableInfo> tablesOneLevel = relation.getBaseTablesOneLevel();
- for (BaseTableInfo info : tablesOneLevel) {
- TRow trow = new TRow();
- trow.addToColumnValue(new TCell().setStringVal(InternalCatalog.INTERNAL_CATALOG_NAME));
- trow.addToColumnValue(new TCell().setStringVal(dbName));
- trow.addToColumnValue(new TCell().setStringVal(tableName));
- trow.addToColumnValue(new TCell().setStringVal(table.getType().name()));
- trow.addToColumnValue(new TCell().setStringVal(info.getCtlName()));
- trow.addToColumnValue(new TCell().setStringVal(info.getDbName()));
- trow.addToColumnValue(new TCell().setStringVal(info.getTableName()));
- trow.addToColumnValue(new TCell().setStringVal(info.getType()));
- dataBatch.add(trow);
- }
+ Stream.concat(relation.getBaseTablesOneLevel().stream(), relation.getBaseViewsOneLevel().stream())
+ .forEach(info -> {
+ TRow trow = new TRow();
+ trow.addToColumnValue(new TCell().setStringVal(InternalCatalog.INTERNAL_CATALOG_NAME));
+ trow.addToColumnValue(new TCell().setStringVal(dbName));
+ trow.addToColumnValue(new TCell().setStringVal(tableName));
+ trow.addToColumnValue(new TCell().setStringVal(table.getType().name()));
+ trow.addToColumnValue(new TCell().setStringVal(info.getCtlName()));
+ trow.addToColumnValue(new TCell().setStringVal(info.getDbName()));
+ trow.addToColumnValue(new TCell().setStringVal(info.getTableName()));
+ trow.addToColumnValue(new TCell().setStringVal(info.getType()));
+ dataBatch.add(trow);
+ });
} else if (table instanceof View) {
String tableName = table.getName();
if (FrontendConjunctsUtils.isFiltered(viewNameConjuncts, "VIEW_NAME", tableName)) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
index d90b43482eb..fe69414a34e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
@@ -137,7 +137,7 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = true;
- relation.getBaseTablesOneLevel();
+ relation.getBaseTablesOneLevelAndFromView();
minTimes = 0;
result = baseTables;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java
index 25e73b37506..ca33a4d6997 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java
@@ -17,15 +17,27 @@
package org.apache.doris.mtmv;
+import org.apache.doris.analysis.StatementBase;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.job.exception.JobException;
+import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.sqltest.SqlTestBase;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
+import org.apache.doris.nereids.trees.plans.commands.info.DistributionDescriptor;
+import org.apache.doris.nereids.trees.plans.commands.info.MTMVPartitionDefinition;
import org.apache.doris.nereids.trees.plans.commands.info.SimpleColumnDefinition;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DecimalV2Type;
@@ -66,7 +78,7 @@ public class MTMVPlanUtilTest extends SqlTestBase {
+ ")\n"
);
- String querySql = "select * from T1";
+ String querySql = "select * from test.T1";
List<ColumnDefinition> actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null,
Sets.newHashSet(), Lists.newArrayList(),
Maps.newHashMap());
@@ -83,14 +95,14 @@ public class MTMVPlanUtilTest extends SqlTestBase {
new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false));
checkRes(expect, actual);
- querySql = "select T1.id from T1 inner join T2 on T1.id = T2.id";
+ querySql = "select T1.id from test.T1 inner join test.T2 on T1.id = T2.id";
actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null,
Sets.newHashSet(), Lists.newArrayList(),
Maps.newHashMap());
expect = Lists.newArrayList(new ColumnDefinition("id", BigIntType.INSTANCE, true));
checkRes(expect, actual);
- querySql = "select id,sum(score) from T1 group by id";
+ querySql = "select id,sum(score) from test.T1 group by id";
actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null,
Sets.newHashSet(), Lists.newArrayList(),
Maps.newHashMap());
@@ -98,7 +110,7 @@ public class MTMVPlanUtilTest extends SqlTestBase {
new ColumnDefinition("__sum_1", BigIntType.INSTANCE, true));
checkRes(expect, actual);
- querySql = "select id,sum(score) from T1 group by id";
+ querySql = "select id,sum(score) from test.T1 group by id";
actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null,
Sets.newHashSet(), Lists.newArrayList(new SimpleColumnDefinition("id", null),
new SimpleColumnDefinition("sum_score", null)),
@@ -107,7 +119,7 @@ public class MTMVPlanUtilTest extends SqlTestBase {
new ColumnDefinition("sum_score", BigIntType.INSTANCE, true));
checkRes(expect, actual);
- querySql = "select * from MTMVPlanUtilTestT1";
+ querySql = "select * from test.MTMVPlanUtilTestT1";
actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null,
Sets.newHashSet(), Lists.newArrayList(),
Maps.newHashMap());
@@ -115,7 +127,7 @@ public class MTMVPlanUtilTest extends SqlTestBase {
new ColumnDefinition("score", StringType.INSTANCE, true));
checkRes(expect, actual);
- querySql = "select score from MTMVPlanUtilTestT1";
+ querySql = "select score from test.MTMVPlanUtilTestT1";
actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null,
Sets.newHashSet(), Lists.newArrayList(),
Maps.newHashMap());
@@ -279,4 +291,209 @@ public class MTMVPlanUtilTest extends SqlTestBase {
Lists.newArrayList(new SimpleColumnDefinition("col1", "c1"),
new SimpleColumnDefinition("col1", "c2")), null));
}
+
+ @Test
+ public void testAnalyzeQuerynNonDeterministic() throws Exception {
+ String querySql = "select *,now() from test.T1";
+ MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition();
+ mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.SELF_MANAGE);
+ DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10,
+ Lists.newArrayList("id"));
+ StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0);
+ LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+
+ AnalysisException exception = Assertions.assertThrows(
+ org.apache.doris.nereids.exceptions.AnalysisException.class, () -> {
+ MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition,
+ distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan);
+ });
+ Assertions.assertTrue(exception.getMessage().contains("nonDeterministic"));
+ }
+
+ @Test
+ public void testAnalyzeQueryFromTablet() throws Exception {
+ String querySql = "select * from test.T1 tablet (20001)";
+ MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition();
+ mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.SELF_MANAGE);
+ DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10,
+ Lists.newArrayList("id"));
+ StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0);
+ LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+
+ AnalysisException exception = Assertions.assertThrows(
+ org.apache.doris.nereids.exceptions.AnalysisException.class, () -> {
+ MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition,
+ distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan);
+ });
+ Assertions.assertTrue(exception.getMessage().contains("invalid expression"));
+ }
+
+ @Test
+ public void testAnalyzeQueryFromTempTable() throws Exception {
+ createTables(
+ "CREATE TEMPORARY TABLE IF NOT EXISTS temp_t1 (\n"
+ + " id varchar(10),\n"
+ + " score String\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "AUTO PARTITION BY LIST(`id`)\n"
+ + "(\n"
+ + ")\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n"
+ );
+ String querySql = "select * from test.temp_t1";
+ MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition();
+ mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.SELF_MANAGE);
+ DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10,
+ Lists.newArrayList("id"));
+ StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0);
+ LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+
+ AnalysisException exception = Assertions.assertThrows(
+ org.apache.doris.nereids.exceptions.AnalysisException.class, () -> {
+ MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition,
+ distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan);
+ });
+ Assertions.assertTrue(exception.getMessage().contains("temporary"));
+ }
+
+ @Test
+ public void testAnalyzeQueryFollowBaseTableFailed() throws Exception {
+ String querySql = "select * from test.T4";
+ MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition();
+ mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE);
+ mtmvPartitionDefinition.setPartitionCol("score");
+ DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10,
+ Lists.newArrayList("id"));
+ StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0);
+ LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+
+ AnalysisException exception = Assertions.assertThrows(
+ org.apache.doris.nereids.exceptions.AnalysisException.class, () -> {
+ MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition,
+ distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan);
+ });
+ Assertions.assertTrue(exception.getMessage().contains("suitable"));
+ }
+
+ @Test
+ public void testAnalyzeQueryNormal() throws Exception {
+ String querySql = "select * from test.T4";
+ MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition();
+ mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE);
+ mtmvPartitionDefinition.setPartitionCol("id");
+ DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10,
+ Lists.newArrayList("id"));
+ StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0);
+ LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan();
+ MTMVAnalyzeQueryInfo mtmvAnalyzeQueryInfo = MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(),
+ querySql, mtmvPartitionDefinition,
+ distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan);
+ Assertions.assertTrue(mtmvAnalyzeQueryInfo.getRelation().getBaseTables().size() == 1);
+ Assertions.assertTrue(mtmvAnalyzeQueryInfo.getMvPartitionInfo().getRelatedCol().equals("id"));
+ Assertions.assertTrue(mtmvAnalyzeQueryInfo.getColumnDefinitions().size() == 2);
+ }
+
+ @Test
+ public void testEnsureMTMVQueryUsable() throws Exception {
+ createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select * from test.T4;");
+ Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
+ MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv1");
+ Assertions.assertDoesNotThrow(
+ () -> MTMVPlanUtil.ensureMTMVQueryUsable(mtmv,
+ MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE)));
+ }
+
+ @Test
+ public void testEnsureMTMVQueryAnalyzeFailed() throws Exception {
+ createTable("CREATE TABLE IF NOT EXISTS analyze_faild_t_partition (\n"
+ + " id bigint not null,\n"
+ + " score bigint\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "AUTO PARTITION BY LIST(`score`)\n"
+ + "(\n"
+ + ")\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n");
+ createTable("CREATE TABLE IF NOT EXISTS analyze_faild_t_not_partition (\n"
+ + " id bigint not null,\n"
+ + " score bigint\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n");
+ createView("create view analyze_faild_v1 as select * from test.analyze_faild_t_partition");
+
+ createMvByNereids("create materialized view analyze_faild_mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ + " PARTITION BY (score)\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select * from test.analyze_faild_v1;");
+ dropView("drop view analyze_faild_v1");
+ createView("create view analyze_faild_v1 as select * from test.analyze_faild_t_not_partition");
+ Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
+ MTMV mtmv = (MTMV) db.getTableOrAnalysisException("analyze_faild_mv1");
+ JobException exception = Assertions.assertThrows(
+ org.apache.doris.job.exception.JobException.class, () -> {
+ MTMVPlanUtil.ensureMTMVQueryUsable(mtmv,
+ MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE));
+ });
+ Assertions.assertTrue(exception.getMessage().contains("suitable"));
+ }
+
+ @Test
+ public void testEnsureMTMVQueryNotEqual() throws Exception {
+ createTable("CREATE TABLE IF NOT EXISTS t_partition1 (\n"
+ + " id bigint not null,\n"
+ + " score bigint\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "AUTO PARTITION BY LIST(`score`)\n"
+ + "(\n"
+ + ")\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n");
+ createTable("CREATE TABLE IF NOT EXISTS t_partition2 (\n"
+ + " id bigint not null,\n"
+ + " score bigint\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "AUTO PARTITION BY LIST(`score`)\n"
+ + "(\n"
+ + ")\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n");
+ createView("create view t_partition1_v1 as select * from test.t_partition1");
+
+ createMvByNereids("create materialized view t_partition1_mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ + " PARTITION BY (score)\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select * from test.t_partition1_v1;");
+ dropView("drop view t_partition1_v1");
+ createView("create view t_partition1_v1 as select * from test.t_partition2");
+ Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test");
+ MTMV mtmv = (MTMV) db.getTableOrAnalysisException("t_partition1_mv1");
+ JobException exception = Assertions.assertThrows(
+ org.apache.doris.job.exception.JobException.class, () -> {
+ MTMVPlanUtil.ensureMTMVQueryUsable(mtmv,
+ MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE));
+ });
+ Assertions.assertTrue(exception.getMessage().contains("changed"));
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java
index 40263705c43..d3c2652208e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java
@@ -95,37 +95,38 @@ public class MTMVRelationManagerTest {
}
@Test
- public void testGetMtmvsByBaseTableOneLevel() {
+ public void testGetMtmvsByBaseTableOneLevelAndFromView() {
// mock mv2==>mv1,t3; mv1==>t4
MTMVRelationManager manager = new MTMVRelationManager();
MTMVRelation mv2Relation = new MTMVRelation(Sets.newHashSet(mv1, t3, t4), Sets.newHashSet(mv1, t3),
- Sets.newHashSet());
- MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4),
- Sets.newHashSet());
+ Sets.newHashSet(mv1, t3),
+ Sets.newHashSet(), Sets.newHashSet());
+ MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4), Sets.newHashSet(t4),
+ Sets.newHashSet(), Sets.newHashSet());
manager.refreshMTMVCache(mv2Relation, mv2);
manager.refreshMTMVCache(mv1Relation, mv1);
// should return mv2
- Set<BaseTableInfo> mv1OneLevel = manager.getMtmvsByBaseTableOneLevel(mv1);
+ Set<BaseTableInfo> mv1OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(mv1);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), mv1OneLevel));
// should return mv2
- Set<BaseTableInfo> t3OneLevel = manager.getMtmvsByBaseTableOneLevel(t3);
+ Set<BaseTableInfo> t3OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(t3);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3OneLevel));
// should return mv1
- Set<BaseTableInfo> t4OneLevel = manager.getMtmvsByBaseTableOneLevel(t4);
+ Set<BaseTableInfo> t4OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(t4);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4OneLevel));
// update mv2 only use t3,remove mv1
- mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3),
- Sets.newHashSet());
+ mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), Sets.newHashSet(t3),
+ Sets.newHashSet(), Sets.newHashSet());
manager.refreshMTMVCache(mv2Relation, mv2);
// should return empty
- mv1OneLevel = manager.getMtmvsByBaseTableOneLevel(mv1);
+ mv1OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(mv1);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(), mv1OneLevel));
// should return mv2
- t3OneLevel = manager.getMtmvsByBaseTableOneLevel(t3);
+ t3OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(t3);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3OneLevel));
// should return mv1
- t4OneLevel = manager.getMtmvsByBaseTableOneLevel(t4);
+ t4OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(t4);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4OneLevel));
}
@@ -134,9 +135,10 @@ public class MTMVRelationManagerTest {
// mock mv2==>mv1,t3; mv1==>t4
MTMVRelationManager manager = new MTMVRelationManager();
MTMVRelation mv2Relation = new MTMVRelation(Sets.newHashSet(mv1, t3, t4), Sets.newHashSet(mv1, t3),
- Sets.newHashSet());
- MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4),
- Sets.newHashSet());
+ Sets.newHashSet(mv1, t3),
+ Sets.newHashSet(), Sets.newHashSet());
+ MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4), Sets.newHashSet(t4),
+ Sets.newHashSet(), Sets.newHashSet());
manager.refreshMTMVCache(mv2Relation, mv2);
manager.refreshMTMVCache(mv1Relation, mv1);
// should return mv2
@@ -150,17 +152,17 @@ public class MTMVRelationManagerTest {
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1, mv2), t4All));
// update mv2 only use t3,remove mv1
- mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3),
- Sets.newHashSet());
+ mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), Sets.newHashSet(t3),
+ Sets.newHashSet(), Sets.newHashSet());
manager.refreshMTMVCache(mv2Relation, mv2);
// should return empty
- mv1All = manager.getMtmvsByBaseTableOneLevel(mv1);
+ mv1All = manager.getMtmvsByBaseTableOneLevelAndFromView(mv1);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(), mv1All));
// should return mv2
- t3All = manager.getMtmvsByBaseTableOneLevel(t3);
+ t3All = manager.getMtmvsByBaseTableOneLevelAndFromView(t3);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3All));
// should return mv1
- t4All = manager.getMtmvsByBaseTableOneLevel(t4);
+ t4All = manager.getMtmvsByBaseTableOneLevelAndFromView(t4);
Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4All));
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java
new file mode 100644
index 00000000000..3b93daca9b7
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mtmv;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.collect.Sets;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class MTMVRelationTest extends TestWithFeService {
+ // t1 => v1 => v2
+ // t2 => mv1
+ // mv1 join v2 => mv2
+ @Test
+ public void testMTMVRelation() throws Exception {
+ createDatabaseAndUse("db1");
+ createTables(
+ "CREATE TABLE IF NOT EXISTS t1 (\n"
+ + " id varchar(10),\n"
+ + " score String\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n",
+ "CREATE TABLE IF NOT EXISTS t2 (\n"
+ + " id varchar(10),\n"
+ + " score String\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n"
+ );
+ createView("create view v1 as select * from t1");
+ createView("create view v2 as select * from v1");
+ createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select * from t2;");
+ createMvByNereids("create materialized view mv2 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select mv1.* from mv1, v2 ;");
+ Database db1 = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("db1");
+ MTMV mtmv = (MTMV) db1.getTableOrAnalysisException("mv2");
+ MTMVRelation relation = mtmv.getRelation();
+ BaseTableInfo t1 = new BaseTableInfo(db1.getTableOrAnalysisException("t1"));
+ BaseTableInfo t2 = new BaseTableInfo(db1.getTableOrAnalysisException("t2"));
+ BaseTableInfo v1 = new BaseTableInfo(db1.getTableOrAnalysisException("v1"));
+ BaseTableInfo v2 = new BaseTableInfo(db1.getTableOrAnalysisException("v2"));
+ BaseTableInfo mv1 = new BaseTableInfo(db1.getTableOrAnalysisException("mv1"));
+ BaseTableInfo mv2 = new BaseTableInfo(db1.getTableOrAnalysisException("mv2"));
+ // test forward index
+ Assertions.assertEquals(Sets.newHashSet(t1, t2, mv1), relation.getBaseTables());
+ Assertions.assertEquals(Sets.newHashSet(v1, v2), relation.getBaseViews());
+ Assertions.assertEquals(Sets.newHashSet(mv1), relation.getBaseTablesOneLevel());
+ Assertions.assertEquals(Sets.newHashSet(v2), relation.getBaseViewsOneLevel());
+ Assertions.assertEquals(Sets.newHashSet(mv1, t1), relation.getBaseTablesOneLevelAndFromView());
+
+ // test inverted index
+ MTMVRelationManager relationManager = Env.getCurrentEnv().getMtmvService().getRelationManager();
+ Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseTable(t1));
+ Assertions.assertEquals(Sets.newHashSet(mv2, mv1), relationManager.getMtmvsByBaseTable(t2));
+ Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseTable(mv1));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(v1));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(v2));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(mv2));
+
+ Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseTableOneLevelAndFromView(t1));
+ Assertions.assertEquals(Sets.newHashSet(mv1), relationManager.getMtmvsByBaseTableOneLevelAndFromView(t2));
+ Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseTableOneLevelAndFromView(mv1));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(v1));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(v2));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(mv2));
+
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(t1));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(t2));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(mv1));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(mv2));
+ Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseView(v1));
+ Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseView(v2));
+
+ dropMvByNereids("drop materialized view mv2");
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(t1));
+ Assertions.assertEquals(Sets.newHashSet(mv1), relationManager.getMtmvsByBaseTable(t2));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(mv1));
+
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(t1));
+ Assertions.assertEquals(Sets.newHashSet(mv1), relationManager.getMtmvsByBaseTableOneLevelAndFromView(t2));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(mv1));
+
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(v1));
+ Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(v2));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVSchemaChangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVSchemaChangeTest.java
new file mode 100644
index 00000000000..e911bd2287f
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVSchemaChangeTest.java
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mtmv;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class MTMVSchemaChangeTest extends TestWithFeService {
+ // t1 => v1 => mv1
+ @Test
+ public void testDropBaseView() throws Exception {
+ createDatabaseAndUse("db1");
+ createTables(
+ "CREATE TABLE IF NOT EXISTS t1 (\n"
+ + " id varchar(10),\n"
+ + " score String\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n"
+ );
+ createView("create view v1 as select * from t1");
+ createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select * from v1 ;");
+ Database db1 = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("db1");
+ MTMV mtmv = (MTMV) db1.getTableOrAnalysisException("mv1");
+ MTMVRefreshSnapshot mtmvRefreshSnapshot = new MTMVRefreshSnapshot();
+ mtmv.setRefreshSnapshot(mtmvRefreshSnapshot);
+ Assertions.assertTrue(mtmv.getRefreshSnapshot().equals(mtmvRefreshSnapshot));
+ Assertions.assertTrue(mtmv.getSchemaChangeVersion() == 0);
+ Assertions.assertTrue(mtmv.getStatus().getState().equals(MTMVState.INIT));
+ dropView("drop view v1");
+ Assertions.assertFalse(mtmv.getRefreshSnapshot().equals(mtmvRefreshSnapshot));
+ Assertions.assertTrue(mtmv.getSchemaChangeVersion() == 1);
+ Assertions.assertTrue(mtmv.getStatus().getState().equals(MTMVState.SCHEMA_CHANGE));
+ }
+
+ @Test
+ public void testDropBaseTable() throws Exception {
+ createDatabaseAndUse("db2");
+ createTables(
+ "CREATE TABLE IF NOT EXISTS t1 (\n"
+ + " id varchar(10),\n"
+ + " score String\n"
+ + ")\n"
+ + "DUPLICATE KEY(id)\n"
+ + "DISTRIBUTED BY HASH(id) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + " \"replication_num\" = \"1\"\n"
+ + ")\n"
+ );
+ createView("create view v1 as select * from t1");
+ createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select * from v1 ;");
+ Database db2 = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("db2");
+ MTMV mtmv = (MTMV) db2.getTableOrAnalysisException("mv1");
+ MTMVRefreshSnapshot mtmvRefreshSnapshot = new MTMVRefreshSnapshot();
+ mtmv.setRefreshSnapshot(mtmvRefreshSnapshot);
+ Assertions.assertTrue(mtmv.getRefreshSnapshot().equals(mtmvRefreshSnapshot));
+ Assertions.assertTrue(mtmv.getSchemaChangeVersion() == 0);
+ Assertions.assertTrue(mtmv.getStatus().getState().equals(MTMVState.INIT));
+ dropTable("t1", true);
+ Assertions.assertTrue(mtmv.getRefreshSnapshot().equals(mtmvRefreshSnapshot));
+ Assertions.assertTrue(mtmv.getSchemaChangeVersion() == 1);
+ Assertions.assertTrue(mtmv.getStatus().getState().equals(MTMVState.SCHEMA_CHANGE));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java
index da9315d8ebc..d2008627b73 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java
@@ -44,7 +44,8 @@ public class MTMVTaskTest {
private String poneName = "p1";
private String ptwoName = "p2";
private List<String> allPartitionNames = Lists.newArrayList(poneName, ptwoName);
- private MTMVRelation relation = new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet());
+ private MTMVRelation relation = new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet(),
+ Sets.newHashSet(), Sets.newHashSet());
@Mocked
private MTMV mtmv;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
index b8c438348ac..5da3a045231 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java
@@ -63,7 +63,7 @@ public class MTMVTest {
+ "finishTimeMs=null, taskType=null, errMsg='null'}]}, mvProperties={}, "
+ "relation=MTMVRelation{baseTables=[], baseTablesOneLevel=[], baseViews=[]}, "
+ "mvPartitionInfo=MTMVPartitionInfo{partitionType=null, relatedTable=null, "
- + "relatedCol='null', partitionCol='null'}, "
+ + "relatedCol='null', partitionCol='null', expr='null'}, "
+ "refreshSnapshot=MTMVRefreshSnapshot{partitionSnapshots={}}, id=1, name='null', "
+ "qualifiedDbName='db1', comment='comment1'}";
MTMV mtmv = new MTMV();
@@ -75,7 +75,8 @@ public class MTMVTest {
mtmv.setStatus(new MTMVStatus());
mtmv.setJobInfo(buildMTMVJobInfo(mtmv));
mtmv.setMvProperties(new HashMap<>());
- mtmv.setRelation(new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet()));
+ mtmv.setRelation(new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet(),
+ Sets.newHashSet()));
mtmv.setMvPartitionInfo(new MTMVPartitionInfo());
mtmv.setRefreshSnapshot(new MTMVRefreshSnapshot());
Assert.assertEquals(expect, mtmv.toInfoString());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
index 2e4361000c9..8e997062f50 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
@@ -148,7 +148,7 @@ public class PlanChecker {
}
public PlanChecker analyze() {
- this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.newTableCollector(true).collect();
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.setCteContext(new CTEContext());
MemoTestUtils.initMemoAndValidState(cascadesContext);
@@ -157,7 +157,7 @@ public class PlanChecker {
public PlanChecker analyze(Plan plan) {
this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan);
- this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.newTableCollector(true).collect();
this.cascadesContext.setCteContext(new CTEContext());
Set<String> originDisableRules = connectContext.getSessionVariable().getDisableNereidsRuleNames();
Set<String> disableRuleWithAuth = Sets.newHashSet(originDisableRules);
@@ -171,7 +171,7 @@ public class PlanChecker {
public PlanChecker analyze(String sql) {
this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql);
- this.cascadesContext.newTableCollector().collect();
+ this.cascadesContext.newTableCollector(true).collect();
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.setCteContext(new CTEContext());
MemoTestUtils.initMemoAndValidState(cascadesContext);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index abb50a7a074..972b18fd946 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -63,6 +63,7 @@ import org.apache.doris.nereids.trees.plans.commands.DropMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.DropRowPolicyCommand;
import org.apache.doris.nereids.trees.plans.commands.DropSqlBlockRuleCommand;
import org.apache.doris.nereids.trees.plans.commands.DropTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.DropViewCommand;
import org.apache.doris.nereids.trees.plans.commands.GrantRoleCommand;
import org.apache.doris.nereids.trees.plans.commands.GrantTablePrivilegeCommand;
import org.apache.doris.nereids.trees.plans.commands.RecoverTableCommand;
@@ -712,6 +713,12 @@ public abstract class TestWithFeService {
command.run(connectContext, new StmtExecutor(connectContext, sql));
}
+ public void dropView(String sql) throws Exception {
+ NereidsParser nereidsParser = new NereidsParser();
+ DropViewCommand command = (DropViewCommand) nereidsParser.parseSingle(sql);
+ command.run(connectContext, new StmtExecutor(connectContext, sql));
+ }
+
protected void createPolicy(String sql) throws Exception {
NereidsParser nereidsParser = new NereidsParser();
CreatePolicyCommand command = (CreatePolicyCommand) nereidsParser.parseSingle(sql);
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out
new file mode 100644
index 00000000000..90c7a97c096
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out
@@ -0,0 +1,30 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !is_sync_init --
+false
+
+-- !is_sync_after_refresh --
+true
+
+-- !after_refresh --
+1 1
+
+-- !not_need_refresh --
+NOT_REFRESH
+
+-- !is_sync_data_change --
+false
+
+-- !trigger_view_need_refresh --
+COMPLETE
+
+-- !after_trigger_view --
+1 1
+2 2
+
+-- !trigger_table_not_need_refresh --
+NOT_REFRESH
+
+-- !after_trigger_table --
+1 1
+2 2
+
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter.out
new file mode 100644
index 00000000000..5ad3271eba9
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_1 --
+1 2
+
+-- !state_alter_view --
+SCHEMA_CHANGE
+
+-- !state_after_refresh --
+NORMAL
+
+-- !need_refresh_complete --
+COMPLETE
+
+-- !select_2 --
+1 3
+
+-- !need_refresh_failed --
+FAILED
+
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter_table.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter_table.out
new file mode 100644
index 00000000000..f4dcc52e9d2
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter_table.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_1 --
+1 1
+
+-- !state_after_drop --
+SCHEMA_CHANGE
+
+-- !need_refresh_failed --
+FAILED
+
+-- !need_refresh_complete --
+COMPLETE
+
+-- !select_2 --
+2 2
+
+-- !state_after_refresh --
+NORMAL
+
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_commit.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_commit.out
new file mode 100644
index 00000000000..0f44c5dce4f
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_commit.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !is_sync_init --
+false
+
+-- !after_refresh --
+1 1
+
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_cte.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_cte.out
new file mode 100644
index 00000000000..ebad07a8c2c
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_cte.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !is_sync_init --
+false
+
+-- !after_refresh --
+1 2 2 3
+
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_mtmv.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_mtmv.out
new file mode 100644
index 00000000000..2cb90afa89e
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_mtmv.out
@@ -0,0 +1,20 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_mv1 --
+1 2017-01-15 1
+1 2017-02-15 2
+1 2017-03-15 3
+
+-- !is_sync_mv2 --
+true
+
+-- !select_mv2 --
+1 2017-01-15 1
+1 2017-02-15 2
+1 2017-03-15 3
+
+-- !is_sync_data_change --
+false
+
+-- !pct_refresh --
+PARTIAL
+
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_pct.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_pct.out
new file mode 100644
index 00000000000..2281daaaadb
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_pct.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !is_sync_init --
+false
+
+-- !is_sync_after_refresh --
+true
+
+-- !after_refresh --
+1 2017-01-15 1
+1 2017-02-15 2
+1 2017-03-15 3
+
+-- !not_need_refresh --
+NOT_REFRESH
+
+-- !is_sync_data_change --
+false
+
+-- !pct_refresh --
+PARTIAL
+
diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_rollup.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_rollup.out
new file mode 100644
index 00000000000..4c30d8bd01a
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_rollup.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !is_sync_after_refresh --
+true
+
+-- !after_refresh --
+1 2017-01-15 1
+1 2017-02-15 2
+1 2017-03-15 3
+
diff --git a/regression-test/data/mtmv_p0/test_limit_partition_view_mtmv.out b/regression-test/data/mtmv_p0/test_limit_partition_view_mtmv.out
new file mode 100644
index 00000000000..a655b5fdeff
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_limit_partition_view_mtmv.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !date_list --
+1 2038-01-01
+
+-- !varchar_list --
+1 20380101
+
+-- !varchar_list --
+1 20380101
+
+-- !date_range --
+1 2038-01-02
+
+-- !date_range_all --
+1 2038-01-02
+2 2020-01-02
+
diff --git a/regression-test/data/mtmv_p0/test_view_hive_mtmv.out b/regression-test/data/mtmv_p0/test_view_hive_mtmv.out
new file mode 100644
index 00000000000..165e9809af8
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_view_hive_mtmv.out
@@ -0,0 +1,28 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !refresh_one_partition --
+1 A 20230101
+2 B 20230101
+3 C 20230101
+
+-- !refresh_complete --
+1 A 20230101
+2 B 20230101
+3 C 20230101
+4 D 20230102
+5 E 20230102
+6 F 20230102
+
+-- !is_sync_before_rebuild --
+true
+
+-- !is_sync_after_rebuild --
+true
+
+-- !refresh_complete_rebuild --
+1 A 20230101
+2 B 20230101
+3 C 20230101
+4 D 20230102
+5 E 20230102
+6 F 20230102
+
diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
index 0c8456fd0a2..32656fc7537 100644
--- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
@@ -146,21 +146,6 @@ suite("test_build_mtmv") {
log.info(e.getMessage())
}
- // not allow create mv use view
- try {
- sql """
- CREATE MATERIALIZED VIEW ${mvNameRenamed}
- BUILD DEFERRED REFRESH COMPLETE ON MANUAL
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES ('replication_num' = '1')
- AS
- SELECT * from ${viewName};
- """
- Assert.fail();
- } catch (Exception e) {
- log.info(e.getMessage())
- }
-
sql """
DROP MATERIALIZED VIEW ${mvName}
"""
diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view.groovy
new file mode 100644
index 00000000000..aab474454d3
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view.groovy
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_create_mtmv_with_view","mtmv") {
+ String suiteName = "test_create_mtmv_with_view"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k2 TINYINT,
+ k3 INT not null
+ )
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * from ${viewName};
+ """
+
+ sql """
+ insert into ${tableName} values(1,1);
+ """
+ order_qt_is_sync_init "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_is_sync_after_refresh "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+ order_qt_after_refresh "SELECT * FROM ${mvName}"
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_not_need_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+
+ sql """
+ insert into ${tableName} values(2,2);
+ """
+ order_qt_is_sync_data_change "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+
+ // test excluded_trigger_tables
+ // view should not useful
+ sql """
+ alter Materialized View ${mvName} set("excluded_trigger_tables"="internal.${dbName}.${viewName}");
+ """
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+
+ order_qt_trigger_view_need_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+ order_qt_after_trigger_view "SELECT * FROM ${mvName}"
+
+ // table should useful
+ sql """
+ alter Materialized View ${mvName} set("excluded_trigger_tables"="internal.${dbName}.${tableName}");
+ """
+ sql """
+ insert into ${tableName} values(3,3);
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+
+ order_qt_trigger_table_not_need_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+ order_qt_after_trigger_table "SELECT * FROM ${mvName}"
+
+
+ sql """drop view if exists `${viewName}`"""
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter.groovy
new file mode 100644
index 00000000000..3973fde6035
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter.groovy
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_create_mtmv_with_view_alter","mtmv") {
+ String suiteName = "test_create_mtmv_with_view_alter"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k2 TINYINT,
+ k3 INT not null
+ )
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql"""
+ create view ${viewName} as select k2,k3+1 as k4 from ${tableName};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * from ${viewName};
+ """
+
+ sql """
+ insert into ${tableName} values(1,1);
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_select_1 "SELECT * FROM ${mvName}"
+ // should refresh all
+ sql"""
+ alter view ${viewName} as select k2,k3+2 as k4 from ${tableName};
+ """
+
+ order_qt_state_alter_view "select State from mv_infos('database'='${dbName}') where Name='${mvName}'"
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_state_after_refresh "select State from mv_infos('database'='${dbName}') where Name='${mvName}'"
+ order_qt_need_refresh_complete "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+ order_qt_select_2 "SELECT * FROM ${mvName}"
+
+ // column k4 not exist, should refresh failed
+ sql"""
+ alter view ${viewName} as select k2,k3+3 as k5 from ${tableName};
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ def jobName = getJobName(dbName, mvName);
+ waitingMTMVTaskFinishedNotNeedSuccess(jobName)
+ order_qt_need_refresh_failed "select Status from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+
+ sql """drop view if exists `${viewName}`"""
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter_table.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter_table.groovy
new file mode 100644
index 00000000000..b8802bf9f9f
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter_table.groovy
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_create_mtmv_with_view_alter_table","mtmv") {
+ String suiteName = "test_create_mtmv_with_view_alter_table"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k2 TINYINT,
+ k3 INT not null
+ )
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * from ${viewName};
+ """
+
+ sql """
+ insert into ${tableName} values(1,1);
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_select_1 "SELECT * FROM ${mvName}"
+
+ sql """
+ drop table ${tableName};
+ """
+ order_qt_state_after_drop "select State from mv_infos('database'='${dbName}') where Name='${mvName}'"
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ def jobName = getJobName(dbName, mvName);
+ waitingMTMVTaskFinishedNotNeedSuccess(jobName)
+ order_qt_need_refresh_failed "select Status from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k2 TINYINT,
+ k3 INT not null
+ )
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """
+ insert into ${tableName} values(2,2);
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_need_refresh_complete "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+ order_qt_select_2 "SELECT * FROM ${mvName}"
+ order_qt_state_after_refresh "select State from mv_infos('database'='${dbName}') where Name='${mvName}'"
+
+ sql """drop view if exists `${viewName}`"""
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_commit.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_commit.groovy
new file mode 100644
index 00000000000..e977e05e33e
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_commit.groovy
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_create_mtmv_with_view_commit","mtmv") {
+ String suiteName = "test_create_mtmv_with_view_commit"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k2 TINYINT,
+ k3 INT not null
+ )
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON COMMIT
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * from ${viewName};
+ """
+
+ order_qt_is_sync_init "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+
+ sql """
+ insert into ${tableName} values(1,1);
+ """
+
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_after_refresh "SELECT * FROM ${mvName}"
+
+ sql """drop view if exists `${viewName}`"""
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_cte.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_cte.groovy
new file mode 100644
index 00000000000..3bc03bd2d2d
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_cte.groovy
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_create_mtmv_with_view_cte","mtmv") {
+ String suiteName = "test_create_mtmv_with_view_cte"
+ String tableName1 = "${suiteName}_table1"
+ String tableName2 = "${suiteName}_table2"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName1}`"""
+ sql """drop table if exists `${tableName2}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE ${tableName1}
+ (
+ k1 INT,
+ k2 INT not null
+ )
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """
+ CREATE TABLE ${tableName2}
+ (
+ k3 INT,
+ k4 INT not null
+ )
+ DISTRIBUTED BY HASH(k4) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql"""
+ create view ${viewName} as select * from ${tableName1};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON COMMIT
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ with cte1 as (
+ select * from ${tableName2}
+ )
+ SELECT * from ${viewName} a inner join cte1 b on a.k2=b.k3;
+ """
+
+ order_qt_is_sync_init "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+
+ sql """
+ insert into ${tableName1} values(1,2);
+ """
+ sql """
+ insert into ${tableName2} values(2,3);
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_after_refresh "SELECT * FROM ${mvName}"
+
+ sql """drop view if exists `${viewName}`"""
+ sql """drop table if exists `${tableName1}`"""
+ sql """drop table if exists `${tableName2}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_mtmv.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_mtmv.groovy
new file mode 100644
index 00000000000..63e83b7890d
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_mtmv.groovy
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_create_mtmv_with_view_mtmv","mtmv") {
+ String suiteName = "test_create_mtmv_with_view_mtmv"
+ String tableName = "${suiteName}_table"
+ String mvName1 = "${suiteName}_mv1"
+ String mvName2 = "${suiteName}_mv2"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName1};"""
+ sql """drop materialized view if exists ${mvName2};"""
+
+ sql """
+ CREATE TABLE `${tableName}` (
+ `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+ `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
+ `num` SMALLINT NOT NULL COMMENT '\"数量\"'
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`user_id`, `date`, `num`)
+ COMMENT 'OLAP'
+ PARTITION BY RANGE(`date`)
+ (PARTITION p201701 VALUES [('0000-01-01'), ('2017-02-01')),
+ PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')),
+ PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01')))
+ DISTRIBUTED BY HASH(`user_id`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+
+ sql """
+ insert into ${tableName} values(1,"2017-01-15",1),(1,"2017-02-15",2),(1,"2017-03-15",3);
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName1}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`date`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * from ${tableName};
+ """
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName1} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName1)
+
+ order_qt_select_mv1 "SELECT * FROM ${mvName1}"
+
+ sql"""
+ create view ${viewName} as select * from ${mvName1};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName2}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`date`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * from ${viewName};
+ """
+
+
+ def showPartitionsResult = sql """show partitions from ${mvName2}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertTrue(showPartitionsResult.toString().contains("p_00000101_20170201"))
+ assertTrue(showPartitionsResult.toString().contains("p_20170201_20170301"))
+ assertTrue(showPartitionsResult.toString().contains("p_20170301_20170401"))
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName2} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName2)
+ order_qt_is_sync_mv2 "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName2}'"
+ order_qt_select_mv2 "SELECT * FROM ${mvName2}"
+
+ sql """
+ insert into ${tableName} values(1,"2017-01-15",4);
+ """
+ order_qt_is_sync_data_change "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName1}'"
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName1} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName1)
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName2} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName2)
+ order_qt_pct_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName2}' order by CreateTime desc limit 1;"
+
+ mv_rewrite_success_without_check_chosen("select * from ${mvName1}", "${mvName2}")
+ mv_rewrite_success_without_check_chosen("select * from ${tableName}", "${mvName1}")
+ mv_rewrite_success_without_check_chosen("select * from ${viewName}", "${mvName2}")
+ // when nest mv is partitioned, rewrite has bug, after fixed, open it
+ // mv_rewrite_success_without_check_chosen("select * from ${tableName}", "${mvName2}")
+
+ sql """drop view if exists `${viewName}`"""
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName1};"""
+ sql """drop materialized view if exists ${mvName2};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_pct.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_pct.groovy
new file mode 100644
index 00000000000..ac4157b91f1
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_pct.groovy
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_create_mtmv_with_view_pct","mtmv") {
+ String suiteName = "test_create_mtmv_with_view_pct"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE `${tableName}` (
+ `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+ `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
+ `num` SMALLINT NOT NULL COMMENT '\"数量\"'
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`user_id`, `date`, `num`)
+ COMMENT 'OLAP'
+ PARTITION BY RANGE(`date`)
+ (PARTITION p201701 VALUES [('0000-01-01'), ('2017-02-01')),
+ PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')),
+ PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01')))
+ DISTRIBUTED BY HASH(`user_id`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+
+ sql """
+ insert into ${tableName} values(1,"2017-01-15",1),(1,"2017-02-15",2),(1,"2017-03-15",3);
+ """
+
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`date`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * from ${viewName};
+ """
+
+ def showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertTrue(showPartitionsResult.toString().contains("p_00000101_20170201"))
+ assertTrue(showPartitionsResult.toString().contains("p_20170201_20170301"))
+ assertTrue(showPartitionsResult.toString().contains("p_20170301_20170401"))
+
+ order_qt_is_sync_init "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_is_sync_after_refresh "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+ order_qt_after_refresh "SELECT * FROM ${mvName}"
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_not_need_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+
+ sql """
+ insert into ${tableName} values(1,"2017-01-15",4);
+ """
+ order_qt_is_sync_data_change "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_pct_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;"
+
+ mv_rewrite_success_without_check_chosen("select * from ${viewName}", "${mvName}")
+ mv_rewrite_success_without_check_chosen("select * from ${tableName}", "${mvName}")
+
+ sql """drop view if exists `${viewName}`"""
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_rollup.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_rollup.groovy
new file mode 100644
index 00000000000..0d65be6382c
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_rollup.groovy
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_create_mtmv_with_view_rollup","mtmv") {
+ String suiteName = "test_create_mtmv_with_view_rollup"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE `${tableName}` (
+ `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+ `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
+ `num` SMALLINT NOT NULL COMMENT '\"数量\"'
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`user_id`, `date`, `num`)
+ COMMENT 'OLAP'
+ PARTITION BY RANGE(`date`)
+ (PARTITION p201701 VALUES [('2017-01-01'), ('2017-02-01')),
+ PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')),
+ PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01')))
+ DISTRIBUTED BY HASH(`user_id`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+
+ sql """
+ insert into ${tableName} values(1,"2017-01-15",1),(1,"2017-02-15",2),(1,"2017-03-15",3);
+ """
+
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by (date_trunc(`date`,'year'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * from ${viewName};
+ """
+
+ def showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertEquals(1, showPartitionsResult.size())
+ assertTrue(showPartitionsResult.toString().contains("p_20170101_20180101"))
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_is_sync_after_refresh "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+ order_qt_after_refresh "SELECT * FROM ${mvName}"
+
+ sql """drop view if exists `${viewName}`"""
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_limit_partition_view_mtmv.groovy b/regression-test/suites/mtmv_p0/test_limit_partition_view_mtmv.groovy
new file mode 100644
index 00000000000..9d399660b6e
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_limit_partition_view_mtmv.groovy
@@ -0,0 +1,254 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_limit_partition_view_mtmv","mtmv") {
+ String suiteName = "test_limit_partition_view_mtmv"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ // list partition date type
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE TABLE `${tableName}` (
+ `k1` LARGEINT NOT NULL COMMENT '\"用户id\"',
+ `k2` DATE NOT NULL COMMENT '\"数据灌入日期时间\"'
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ COMMENT 'OLAP'
+ PARTITION BY list(`k2`)
+ (
+ PARTITION p_20380101 VALUES IN ("2038-01-01"),
+ PARTITION p_20200101 VALUES IN ("2020-01-01")
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+ sql """
+ insert into ${tableName} values(1,"2038-01-01"),(2,"2020-01-01");
+ """
+
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`k2`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'partition_sync_limit'='2',
+ 'partition_sync_time_unit'='YEAR'
+ )
+ AS
+ SELECT * FROM ${viewName};
+ """
+ def showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertEquals(1, showPartitionsResult.size())
+ assertTrue(showPartitionsResult.toString().contains("p_20380101"))
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_date_list "SELECT * FROM ${mvName} order by k1,k2"
+
+
+
+ // list partition string type
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE TABLE `${tableName}` (
+ `k1` LARGEINT NOT NULL COMMENT '\"用户id\"',
+ `k2` VARCHAR(100) NOT NULL COMMENT '\"数据灌入日期时间\"'
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ COMMENT 'OLAP'
+ PARTITION BY list(`k2`)
+ (
+ PARTITION p_20380101 VALUES IN ("20380101"),
+ PARTITION p_20200101 VALUES IN ("20200101")
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+ sql """
+ insert into ${tableName} values(1,"20380101"),(2,"20200101");
+ """
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`k2`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'partition_sync_limit'='2',
+ 'partition_sync_time_unit'='MONTH',
+ 'partition_date_format'='%Y%m%d'
+ )
+ AS
+ SELECT * FROM ${viewName};
+ """
+ showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertEquals(1, showPartitionsResult.size())
+ assertTrue(showPartitionsResult.toString().contains("p_20380101"))
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_varchar_list "SELECT * FROM ${mvName} order by k1,k2"
+
+
+ // list partition int type
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE TABLE `${tableName}` (
+ `k1` LARGEINT NOT NULL COMMENT '\"用户id\"',
+ `k2` int NOT NULL COMMENT '\"数据灌入日期时间\"'
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ COMMENT 'OLAP'
+ PARTITION BY list(`k2`)
+ (
+ PARTITION p_20380101 VALUES IN ("20380101"),
+ PARTITION p_20200101 VALUES IN ("20200101")
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+ sql """
+ insert into ${tableName} values(1,20380101),(2,20200101);
+ """
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`k2`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'partition_sync_limit'='2',
+ 'partition_sync_time_unit'='DAY',
+ 'partition_date_format'='%Y%m%d'
+ )
+ AS
+ SELECT * FROM ${viewName};
+ """
+ showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertEquals(1, showPartitionsResult.size())
+ assertTrue(showPartitionsResult.toString().contains("p_20380101"))
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_varchar_list "SELECT * FROM ${mvName} order by k1,k2"
+
+
+ // range partition date type
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE TABLE `${tableName}` (
+ `k1` LARGEINT NOT NULL COMMENT '\"用户id\"',
+ `k2` DATE NOT NULL COMMENT '\"数据灌入日期时间\"'
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ COMMENT 'OLAP'
+ PARTITION BY range(`k2`)
+ (
+ PARTITION p2038 VALUES [("2038-01-01"),("2038-01-03")),
+ PARTITION p2020 VALUES [("2020-01-01"),("2020-01-03"))
+ )
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 2
+ PROPERTIES ('replication_num' = '1') ;
+ """
+ sql """
+ insert into ${tableName} values(1,"2038-01-02"),(2,"2020-01-02");
+ """
+ sql"""
+ create view ${viewName} as select * from ${tableName};
+ """
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`k2`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'partition_sync_limit'='2',
+ 'partition_sync_time_unit'='YEAR'
+ )
+ AS
+ SELECT * FROM ${viewName};
+ """
+ showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertEquals(1, showPartitionsResult.size())
+ assertTrue(showPartitionsResult.toString().contains("p_20380101_20380103"))
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_date_range "SELECT * FROM ${mvName} order by k1,k2"
+
+
+ // alter
+ sql """
+ alter Materialized View ${mvName} set("partition_sync_limit"="");
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} AUTO
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertEquals(2, showPartitionsResult.size())
+ assertTrue(showPartitionsResult.toString().contains("p_20380101_20380103"))
+ assertTrue(showPartitionsResult.toString().contains("p_20200101_20200103"))
+ order_qt_date_range_all "SELECT * FROM ${mvName} order by k1,k2"
+
+ sql """drop table if exists `${tableName}`"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
diff --git a/regression-test/suites/mtmv_p0/test_view_hive_mtmv.groovy b/regression-test/suites/mtmv_p0/test_view_hive_mtmv.groovy
new file mode 100644
index 00000000000..98472d80746
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_view_hive_mtmv.groovy
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_view_hive_mtmv", "p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("diable Hive test.")
+ return;
+ }
+ String suiteName = "test_create_mtmv_with_view_rollup"
+ String mvName = "${suiteName}_mv"
+ String viewName = "${suiteName}_view"
+ String dbName = context.config.getDbNameByFile(context.file)
+ for (String hivePrefix : ["hive3"]) {
+ try {
+ String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+ String catalog_name = "${suiteName}_${hivePrefix}_test_catalog"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+ );"""
+
+ sql"""
+ create view ${viewName} as select * from ${catalog_name}.`default`.mtmv_base1;
+ """
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`part_col`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT * FROM ${viewName};
+ """
+ def showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertTrue(showPartitionsResult.toString().contains("p_20230101"))
+ assertTrue(showPartitionsResult.toString().contains("p_20230102"))
+
+ // refresh one partitions
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20230101);
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_refresh_one_partition "SELECT * FROM ${mvName} order by id"
+
+ //refresh complete
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} auto
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_refresh_complete "SELECT * FROM ${mvName} order by id"
+
+ order_qt_is_sync_before_rebuild "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+ // rebuild catalog, should not Affects MTMV
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+ );"""
+
+ order_qt_is_sync_after_rebuild "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'"
+
+ // should refresh normal after catalog rebuild
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} complete
+ """
+ waitingMTMVTaskFinishedByMvName(mvName)
+ order_qt_refresh_complete_rebuild "SELECT * FROM ${mvName} order by id"
+
+ sql """drop materialized view if exists ${mvName};"""
+ sql """drop view if exists `${viewName}`"""
+ sql """drop catalog if exists ${catalog_name}"""
+ } finally {
+ }
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org