You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2024/07/16 03:46:48 UTC
(doris) branch branch-3.0 updated: [improvement](mv) Catch exception when init sync materialized view context #37701 (#37872)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 2d7acd01bea [improvement](mv) Catch exception when init sync materialized view context #37701 (#37872)
2d7acd01bea is described below
commit 2d7acd01beab07e8b9720d7cb7e40e205de4c927
Author: seawinde <14...@users.noreply.github.com>
AuthorDate: Tue Jul 16 11:46:40 2024 +0800
[improvement](mv) Catch exception when init sync materialized view context #37701 (#37872)
## Proposed changes
cherry-pick 3.0
pr: https://github.com/apache/doris/pull/37701
commit: e478570e
pr: https://github.com/apache/doris/pull/37154
commit: 841e39a2
---------
Co-authored-by: starocean999 <40...@users.noreply.github.com>
---
.../java/org/apache/doris/catalog/OlapTable.java | 9 +++
.../mv/InitMaterializationContextHook.java | 64 ++++++++++++----------
2 files changed, 45 insertions(+), 28 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 7d3b9baf341..471cbdf0ec6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2990,4 +2990,13 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc
public long getReplicaCount() {
return statistics.getReplicaCount();
}
+
+ public boolean isShadowIndex(long indexId) {
+ String indexName = getIndexNameById(indexId);
+ if (indexName != null && indexName.startsWith(org.apache.doris.alter.SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 072765fb2a4..53a352e2ef8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -145,38 +145,46 @@ public class InitMaterializationContextHook implements PlannerHook {
keyCount += column.isKey() ? 1 : 0;
}
for (Map.Entry<String, Long> entry : olapTable.getIndexNameToId().entrySet()) {
- if (entry.getValue() != baseIndexId) {
- MaterializedIndexMeta meta = olapTable.getIndexMetaByIndexId(entry.getValue());
- String createMvSql;
- if (meta.getDefineStmt() != null) {
- // get the original create mv sql
- createMvSql = meta.getDefineStmt().originStmt;
- } else {
- // it's rollup, need assemble create mv sql manually
- if (olapTable.getKeysType() == KeysType.AGG_KEYS) {
- createMvSql = assembleCreateMvSqlForAggTable(olapTable.getQualifiedName(),
- entry.getKey(), meta.getSchema(false), keyCount);
+ long indexId = entry.getValue();
+ try {
+ // when doing schema change, a shadow index would be created and put together with mv indexes
+ // we must roll out these unexpected shadow indexes here
+ if (indexId != baseIndexId && !olapTable.isShadowIndex(indexId)) {
+ MaterializedIndexMeta meta = olapTable.getIndexMetaByIndexId(entry.getValue());
+ String createMvSql;
+ if (meta.getDefineStmt() != null) {
+ // get the original create mv sql
+ createMvSql = meta.getDefineStmt().originStmt;
} else {
- createMvSql =
- assembleCreateMvSqlForDupOrUniqueTable(olapTable.getQualifiedName(),
- entry.getKey(), meta.getSchema(false));
+ // it's rollup, need assemble create mv sql manually
+ if (olapTable.getKeysType() == KeysType.AGG_KEYS) {
+ createMvSql = assembleCreateMvSqlForAggTable(olapTable.getQualifiedName(),
+ entry.getKey(), meta.getSchema(false), keyCount);
+ } else {
+ createMvSql =
+ assembleCreateMvSqlForDupOrUniqueTable(olapTable.getQualifiedName(),
+ entry.getKey(), meta.getSchema(false));
+ }
}
- }
- if (createMvSql != null) {
- Optional<String> querySql =
- new NereidsParser().parseForSyncMv(createMvSql);
- if (!querySql.isPresent()) {
- LOG.warn(String.format("can't parse %s ", createMvSql));
- continue;
+ if (createMvSql != null) {
+ Optional<String> querySql =
+ new NereidsParser().parseForSyncMv(createMvSql);
+ if (!querySql.isPresent()) {
+ LOG.warn(String.format("can't parse %s ", createMvSql));
+ continue;
+ }
+ MTMVCache mtmvCache = MaterializedViewUtils.createMTMVCache(querySql.get(),
+ cascadesContext.getConnectContext());
+ contexts.add(new SyncMaterializationContext(mtmvCache.getLogicalPlan(),
+ mtmvCache.getOriginalPlan(), olapTable, meta.getIndexId(), entry.getKey(),
+ cascadesContext, mtmvCache.getStatistics()));
+ } else {
+ LOG.warn(String.format("can't assemble create mv sql for index ", entry.getKey()));
}
- MTMVCache mtmvCache = MaterializedViewUtils.createMTMVCache(querySql.get(),
- cascadesContext.getConnectContext());
- contexts.add(new SyncMaterializationContext(mtmvCache.getLogicalPlan(),
- mtmvCache.getOriginalPlan(), olapTable, meta.getIndexId(), entry.getKey(),
- cascadesContext, mtmvCache.getStatistics()));
- } else {
- LOG.warn(String.format("can't assemble create mv sql for index ", entry.getKey()));
}
+ } catch (Exception exception) {
+ LOG.warn(String.format("createSyncMvContexts exception, index id is %s, index name is %s",
+ entry.getValue(), entry.getValue()), exception);
}
}
return contexts;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org