You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/08/05 13:47:26 UTC
[incubator-doris] branch master updated: [Alter]Analyze define expr
before replay Rollup job (#4236)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 173bc09 [Alter]Analyze define expr before replay Rollup job (#4236)
173bc09 is described below
commit 173bc09833aa62c5bf08e02e23a57c0b7a91e75f
Author: EmmyMiao87 <52...@qq.com>
AuthorDate: Wed Aug 5 21:47:18 2020 +0800
[Alter]Analyze define expr before replay Rollup job (#4236)
The define expr should be analyzed after replay RollupJob.
The slot desc of define expr is used to transfrom to thrift and send to backend.
---
.../java/org/apache/doris/alter/RollupJobV2.java | 31 +++++++++++++++-------
.../org/apache/doris/alter/RollupJobV2Test.java | 12 ++++++---
2 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 4ee5870..81c25a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -17,8 +17,10 @@
package org.apache.doris.alter;
+import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.MVColumnItem;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.catalog.Catalog;
@@ -45,8 +47,10 @@ import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.SqlModeHelper;
+import org.apache.doris.system.SystemInfoService;
import org.apache.doris.task.AgentBatchTask;
import org.apache.doris.task.AgentTask;
import org.apache.doris.task.AgentTaskExecutor;
@@ -731,11 +735,11 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
this.jobState = jobState;
}
- private void setColumnsDefineExpr(Map<String, Expr> columnNameToDefineExpr) {
- for (Entry<String, Expr> entry : columnNameToDefineExpr.entrySet()) {
+ private void setColumnsDefineExpr(List<MVColumnItem> mvColumnItemList) {
+ for (MVColumnItem mvColumnItem : mvColumnItemList) {
for (Column column : rollupSchema) {
- if (column.getName().equals(entry.getKey())) {
- column.setDefineExpr(entry.getValue());
+ if (column.getName().equals(mvColumnItem.getName())) {
+ column.setDefineExpr(mvColumnItem.getDefineExpr());
break;
}
}
@@ -805,16 +809,25 @@ public class RollupJobV2 extends AlterJobV2 implements GsonPostProcessable {
if (origStmt == null) {
return;
}
+
+ if (jobState != JobState.PENDING) {
+ return;
+ }
// parse the define stmt to schema
SqlParser parser = new SqlParser(new SqlScanner(new StringReader(origStmt.originStmt),
- SqlModeHelper.MODE_DEFAULT));
- CreateMaterializedViewStmt stmt;
+ SqlModeHelper.MODE_DEFAULT));
+ ConnectContext connectContext = new ConnectContext();
+ connectContext.setCluster(SystemInfoService.DEFAULT_CLUSTER);
+ connectContext.setDatabase(Catalog.getCurrentCatalog().getDb(dbId).getFullName());
+ Analyzer analyzer = new Analyzer(Catalog.getCurrentCatalog(), connectContext);
+ CreateMaterializedViewStmt stmt = null;
try {
stmt = (CreateMaterializedViewStmt) SqlParserUtils.getStmt(parser, origStmt.idx);
- Map<String, Expr> columnNameToDefineExpr = stmt.parseDefineExprWithoutAnalyze();
- setColumnsDefineExpr(columnNameToDefineExpr);
+ stmt.analyze(analyzer);
} catch (Exception e) {
- throw new IOException("error happens when parsing create materialized view stmt: " + origStmt, e);
+ // Under normal circumstances, the stmt will not fail to analyze.
+ throw new IOException("error happens when parsing create materialized view stmt: " + stmt, e);
}
+ setColumnsDefineExpr(stmt.getMVColumnItemList());
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
index 6fb4d06..7293740 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/RollupJobV2Test.java
@@ -28,6 +28,7 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.FunctionName;
+import org.apache.doris.analysis.MVColumnItem;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.AggregateType;
@@ -371,6 +372,7 @@ public class RollupJobV2Test {
@Test
public void testSerializeOfRollupJob(@Mocked CreateMaterializedViewStmt stmt) throws IOException,
AnalysisException {
+ Config.enable_materialized_view = true;
// prepare file
File file = new File(fileName);
file.createNewFile();
@@ -395,12 +397,14 @@ public class RollupJobV2Test {
List<Expr> params = Lists.newArrayList();
SlotRef param1 = new SlotRef(new TableName(null, "test"), "c1");
params.add(param1);
- Map<String, Expr> columnNameToDefineExpr = Maps.newHashMap();
- columnNameToDefineExpr.put(mvColumnName, new FunctionCallExpr(new FunctionName("to_bitmap"), params));
+ MVColumnItem mvColumnItem = new MVColumnItem(mvColumnName, Type.BITMAP);
+ mvColumnItem.setDefineExpr(new FunctionCallExpr(new FunctionName("to_bitmap"), params));
+ List<MVColumnItem> mvColumnItemList = Lists.newArrayList();
+ mvColumnItemList.add(mvColumnItem);
new Expectations() {
{
- stmt.parseDefineExprWithoutAnalyze();
- result = columnNameToDefineExpr;
+ stmt.getMVColumnItemList();
+ result = mvColumnItemList;
}
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org