You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/14 08:29:28 UTC
[doris] branch master updated: [Bugfix](Mtmv)Fix mtmv meta load failed (#18605)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5d1abe4507 [Bugfix](Mtmv)Fix mtmv meta load failed (#18605)
5d1abe4507 is described below
commit 5d1abe4507a33f51439346e11f96c16e775ad103
Author: huangzhaowei <ca...@gmail.com>
AuthorDate: Fri Apr 14 16:29:18 2023 +0800
[Bugfix](Mtmv)Fix mtmv meta load failed (#18605)
MTMV meta load fail since meta was public to the CI System
---
.../main/java/org/apache/doris/common/Config.java | 4 ++--
.../main/java/org/apache/doris/alter/Alter.java | 3 +--
.../CreateMultiTableMaterializedViewStmt.java | 5 +++++
.../main/java/org/apache/doris/catalog/Env.java | 12 ++++------
.../apache/doris/datasource/InternalCatalog.java | 5 ++---
.../java/org/apache/doris/mtmv/MTMVJobManager.java | 26 +++++++++-------------
.../org/apache/doris/persist/meta/MetaFooter.java | 2 +-
.../org/apache/doris/persist/meta/MetaReader.java | 12 ++++++++++
.../catalog/MultiTableMaterializedViewTest.java | 2 ++
regression-test/pipeline/p0/conf/fe.conf | 4 ++--
regression-test/pipeline/p1/conf/fe.conf | 4 ++--
11 files changed, 43 insertions(+), 36 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index f38fc90920..97bf1c0e0b 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1825,10 +1825,10 @@ public class Config extends ConfigBase {
public static int statistic_task_scheduler_execution_interval_ms = 1000;
/*
- * mtmv scheduler framework is still under dev, remove this config when it is graduate.
+ * mtmv is still under dev, remove this config when it is graduate.
*/
@ConfField(mutable = true, masterOnly = true)
- public static boolean enable_mtmv_scheduler_framework = false;
+ public static boolean enable_mtmv = false;
/* Max running task num at the same time, otherwise the submitted task will still be keep in pending poll*/
@ConfField(mutable = true, masterOnly = true)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index c8baa592e9..b81dbb726e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -60,7 +60,6 @@ import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.UserException;
@@ -525,7 +524,7 @@ public class Alter {
if (!isReplay) {
Env.getCurrentEnv().getEditLog().logAlterMTMV(alterView);
// 5. master node generate new jobs
- if (Config.enable_mtmv_scheduler_framework && MTMVJobFactory.isGenerateJob(olapTable)) {
+ if (MTMVJobFactory.isGenerateJob(olapTable)) {
List<MTMVJob> jobs = MTMVJobFactory.buildJob(olapTable, db.getFullName());
for (MTMVJob job : jobs) {
Env.getCurrentEnv().getMTMVJobManager().createJob(job, false);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMultiTableMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMultiTableMaterializedViewStmt.java
index 4b1cea4e42..94251bfc82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMultiTableMaterializedViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMultiTableMaterializedViewStmt.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
@@ -61,6 +62,10 @@ public class CreateMultiTableMaterializedViewStmt extends CreateTableStmt {
@Override
public void analyze(Analyzer analyzer) throws UserException {
+ if (!Config.enable_mtmv) {
+ throw new UserException("Multi table materialized view was not graduated."
+ + " You should set `enable_mtmv = true` in fe to enabled it manually.");
+ }
refreshInfo.analyze(analyzer);
queryStmt.setNeedToSql(true);
queryStmt.setToSQLWithHint(true);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 0c79634103..d42dc61451 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -1946,10 +1946,8 @@ public class Env {
* Load mtmv jobManager.
**/
public long loadMTMVJobManager(DataInputStream in, long checksum) throws IOException {
- if (Config.enable_mtmv_scheduler_framework) {
- this.mtmvJobManager = MTMVJobManager.read(in, checksum);
- LOG.info("finished replay mtmv job and tasks from image");
- }
+ this.mtmvJobManager = MTMVJobManager.read(in, checksum);
+ LOG.info("finished replay mtmv job and tasks from image");
return checksum;
}
@@ -2197,10 +2195,8 @@ public class Env {
}
public long saveMTMVJobManager(CountingDataOutputStream out, long checksum) throws IOException {
- if (Config.enable_mtmv_scheduler_framework) {
- Env.getCurrentEnv().getMTMVJobManager().write(out, checksum);
- LOG.info("Save mtmv job and tasks to image");
- }
+ Env.getCurrentEnv().getMTMVJobManager().write(out, checksum);
+ LOG.info("Save mtmv job and tasks to image");
return checksum;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 5b745bca6b..ae443de561 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -974,7 +974,7 @@ public class InternalCatalog implements CatalogIf<Database> {
}
}
- if (table instanceof MaterializedView && Config.enable_mtmv_scheduler_framework) {
+ if (table instanceof MaterializedView) {
List<Long> dropIds = Env.getCurrentEnv().getMTMVJobManager().showJobs(db.getFullName(), table.getName())
.stream().map(MTMVJob::getId).collect(Collectors.toList());
Env.getCurrentEnv().getMTMVJobManager().dropJobs(dropIds, isReplay);
@@ -2323,8 +2323,7 @@ public class InternalCatalog implements CatalogIf<Database> {
throw e;
}
- if (olapTable instanceof MaterializedView && Config.enable_mtmv_scheduler_framework
- && MTMVJobFactory.isGenerateJob((MaterializedView) olapTable)) {
+ if (olapTable instanceof MaterializedView && MTMVJobFactory.isGenerateJob((MaterializedView) olapTable)) {
List<MTMVJob> jobs = MTMVJobFactory.buildJob((MaterializedView) olapTable, db.getFullName());
for (MTMVJob job : jobs) {
Env.getCurrentEnv().getMTMVJobManager().createJob(job, false);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
index c6ff81c082..0e27a4616f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
@@ -45,7 +45,6 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInputStream;
import java.io.DataOutputStream;
-import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -533,25 +532,20 @@ public class MTMVJobManager {
public static MTMVJobManager read(DataInputStream dis, long checksum) throws IOException {
MTMVJobManager mtmvJobManager = new MTMVJobManager();
- try {
- String s = Text.readString(dis);
- MTMVCheckpointData data = GsonUtils.GSON.fromJson(s, MTMVCheckpointData.class);
- if (data != null) {
- if (data.jobs != null) {
- for (MTMVJob job : data.jobs) {
- mtmvJobManager.replayCreateJob(job);
- }
+ String s = Text.readString(dis);
+ MTMVCheckpointData data = GsonUtils.GSON.fromJson(s, MTMVCheckpointData.class);
+ if (data != null) {
+ if (data.jobs != null) {
+ for (MTMVJob job : data.jobs) {
+ mtmvJobManager.replayCreateJob(job);
}
+ }
- if (data.tasks != null) {
- for (MTMVTask runStatus : data.tasks) {
- mtmvJobManager.replayCreateJobTask(runStatus);
- }
+ if (data.tasks != null) {
+ for (MTMVTask runStatus : data.tasks) {
+ mtmvJobManager.replayCreateJobTask(runStatus);
}
}
- LOG.info("finished replaying JobManager from image");
- } catch (EOFException e) {
- LOG.info("no job or task to replay.");
}
return mtmvJobManager;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaFooter.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaFooter.java
index 0857e30909..dcf2fc68b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaFooter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaFooter.java
@@ -45,7 +45,7 @@ import java.util.List;
public class MetaFooter {
private static final Logger LOG = LogManager.getLogger(MetaFooter.class);
- private static final long FOOTER_LENGTH_SIZE = 8L;
+ public static final long FOOTER_LENGTH_SIZE = 8L;
private static final long CHECKSUM_LENGTH_SIZE = 8L;
// checksum
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java
index 13467889e0..8379850de1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaReader.java
@@ -74,6 +74,8 @@ public class MetaReader {
MetaFooter metaFooter = MetaFooter.read(imageFile);
long checksum = 0;
+ long footerIndex = imageFile.length()
+ - metaFooter.length - MetaFooter.FOOTER_LENGTH_SIZE - MetaMagicNumber.MAGIC_STR.length();
try (DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(imageFile)))) {
// 1. Skip image file header
IOUtils.skipFully(dis, metaHeader.getEnd());
@@ -87,6 +89,16 @@ public class MetaReader {
// skip meta header, which has been read before.
continue;
}
+ if (i < metaFooter.metaIndices.size() - 1
+ && metaIndex.offset == metaFooter.metaIndices.get(i + 1).offset) {
+ // skip empty meta
+ LOG.info("Skip {} module since empty meta length.", metaIndex.name);
+ continue;
+ } else if (metaIndex.offset == footerIndex) {
+ // skip last empty meta
+ LOG.info("Skip {} module since empty meta length in the end.", metaIndex.name);
+ continue;
+ }
// Should skip some bytes because ignore some meta, such as load job
if (metaIndex.name.equals("loadJob")
|| metaIndex.name.equals("cooldownJob")) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java
index e4186a86aa..4155d5914b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.MVRefreshInfo;
import org.apache.doris.analysis.MVRefreshIntervalTriggerInfo;
import org.apache.doris.analysis.ShowStmt;
import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
@@ -53,6 +54,7 @@ public class MultiTableMaterializedViewTest extends TestWithFeService {
createDatabase("test");
connectContext.setDatabase("default_cluster:test");
connectContext.getState().reset();
+ Config.enable_mtmv = true;
}
@AfterEach
diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf
index caa1ab7b2b..c11c6fe566 100644
--- a/regression-test/pipeline/p0/conf/fe.conf
+++ b/regression-test/pipeline/p0/conf/fe.conf
@@ -77,5 +77,5 @@ use_fuzzy_session_variable=true
enable_map_type=true
enable_struct_type=true
-# enable mtmv job
-enable_mtmv_scheduler_framework = true
+# enable mtmv
+enable_mtmv = true
diff --git a/regression-test/pipeline/p1/conf/fe.conf b/regression-test/pipeline/p1/conf/fe.conf
index 623b5e5a4c..33d0cd7b25 100644
--- a/regression-test/pipeline/p1/conf/fe.conf
+++ b/regression-test/pipeline/p1/conf/fe.conf
@@ -75,5 +75,5 @@ remote_fragment_exec_timeout_ms=60000
fuzzy_test_type=p1
use_fuzzy_session_variable=true
-# enable mtmv job
-enable_mtmv_scheduler_framework = true
+# enable mtmv
+enable_mtmv = true
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org