You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/08/21 02:42:49 UTC
[incubator-doris] branch master updated: [MV] Input correct keys
type of index meta when `Add Partition` (#4408)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 76a04de [MV] Input correct keys type of index meta when `Add Partition` (#4408)
76a04de is described below
commit 76a04de6c44ab959d3f5c32e7711b5e429beb98d
Author: EmmyMiao87 <52...@qq.com>
AuthorDate: Fri Aug 21 10:42:41 2020 +0800
[MV] Input correct keys type of index meta when `Add Partition` (#4408)
Define Expr will not serialized in Column `toThrift`.
1. When adding partition, different indexes should use their own keys type
instead of using the keys type of base table uniformly.
`
2. There are two kinds of define expr in Column , one is analyzed, and the other is not analyzed.
Currently, analyzed define expr is only used when creating materialized views, so the define expr in RollupJob must be analyzed.
In other cases, such as define expr in `MaterializedIndexMeta`, it may not be analyzed after being relayed.
When executing the load, the analyzed define expr (such as to_bitmap(cast(k1, varchar))) will not be analyzed again.
Only a cast function will be added to the inner layer(such as to_bitmap(cast(cast(k1 ,int), varchar))) which is analyzed too.
The define expr that has not been analyzed (such as cast(k1, varchar)) will be analyzed when executing the load.
---
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java | 2 +-
.../org/apache/doris/analysis/CreateMaterializedViewStmt.java | 2 --
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java | 8 ++------
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java | 9 ++++++---
4 files changed, 9 insertions(+), 12 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index 3094288..4ee4741 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -72,7 +72,7 @@ public class CastExpr extends Expr {
/**
* Copy c'tor used in clone().
*/
- protected CastExpr(TypeDef targetTypeDef, Expr e) {
+ public CastExpr(TypeDef targetTypeDef, Expr e) {
Preconditions.checkNotNull(targetTypeDef);
Preconditions.checkNotNull(e);
this.targetTypeDef = targetTypeDef;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index 7e73bcf..49fffa9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -433,7 +433,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
case FunctionSet.BITMAP_UNION:
if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) {
CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef);
- castExpr.setImplicit(true);
List<Expr> params = Lists.newArrayList();
params.add(castExpr);
FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.TO_BITMAP, params);
@@ -445,7 +444,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
case FunctionSet.HLL_UNION:
if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) {
CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef);
- castExpr.setImplicit(true);
List<Expr> params = Lists.newArrayList();
params.add(castExpr);
FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.HLL_HASH, params);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index be45a50..54ebcf4 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -3173,7 +3173,6 @@ public class Catalog {
olapTable.getBaseIndexId(),
partitionId, partitionName,
indexIdToMeta,
- olapTable.getKeysType(),
distributionInfo,
dataProperty.getStorageMedium(),
singlePartitionDesc.getReplicationNum(),
@@ -3395,7 +3394,6 @@ public class Catalog {
private Partition createPartitionWithIndices(String clusterName, long dbId, long tableId,
long baseIndexId, long partitionId, String partitionName,
Map<Long, MaterializedIndexMeta> indexIdToMeta,
- KeysType keysType,
DistributionInfo distributionInfo,
TStorageMedium storageMedium,
short replicationNum,
@@ -3453,6 +3451,7 @@ public class Catalog {
short shortKeyColumnCount = indexMeta.getShortKeyColumnCount();
TStorageType storageType = indexMeta.getStorageType();
List<Column> schema = indexMeta.getSchema();
+ KeysType keysType = indexMeta.getKeysType();
int totalTaskNum = index.getTablets().size() * replicationNum;
MarkedCountDownLatch<Long, Long> countDownLatch = new MarkedCountDownLatch<Long, Long>(totalTaskNum);
AgentBatchTask batchTask = new AgentBatchTask();
@@ -3744,7 +3743,6 @@ public class Catalog {
olapTable.getId(), olapTable.getBaseIndexId(),
partitionId, partitionName,
olapTable.getIndexIdToMeta(),
- keysType,
distributionInfo,
partitionInfo.getDataProperty(partitionId).getStorageMedium(),
partitionInfo.getReplicationNum(partitionId),
@@ -3773,8 +3771,7 @@ public class Catalog {
DataProperty dataProperty = rangePartitionInfo.getDataProperty(entry.getValue());
Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(), olapTable.getId(),
olapTable.getBaseIndexId(), entry.getValue(), entry.getKey(),
- olapTable.getIndexIdToMeta(),
- keysType, distributionInfo,
+ olapTable.getIndexIdToMeta(), distributionInfo,
dataProperty.getStorageMedium(),
partitionInfo.getReplicationNum(entry.getValue()),
versionInfo, bfColumns, bfFpp,
@@ -6286,7 +6283,6 @@ public class Catalog {
db.getId(), copiedTbl.getId(), copiedTbl.getBaseIndexId(),
newPartitionId, entry.getKey(),
copiedTbl.getIndexIdToMeta(),
- copiedTbl.getKeysType(),
copiedTbl.getDefaultDistributionInfo(),
copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).getStorageMedium(),
copiedTbl.getPartitionInfo().getReplicationNum(oldPartitionId),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 63aa2c5..f0b0a3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -72,6 +72,9 @@ public class Column implements Writable {
private String comment;
@SerializedName(value = "stats")
private ColumnStats stats; // cardinality and selectivity etc.
+ // Define expr may exist in two forms, one is analyzed, and the other is not analyzed.
+ // Currently, analyzed define expr is only used when creating materialized views, so the define expr in RollupJob must be analyzed.
+ // In other cases, such as define expr in `MaterializedIndexMeta`, it may not be analyzed after being relayed.
private Expr defineExpr; // use to define column in materialize view
public Column() {
@@ -259,9 +262,9 @@ public class Column implements Writable {
tColumn.setIs_key(this.isKey);
tColumn.setIs_allow_null(this.isAllowNull);
tColumn.setDefault_value(this.defaultValue);
- if (this.defineExpr != null) {
- tColumn.setDefine_expr(this.defineExpr.treeToThrift());
- }
+ // The define expr does not need to be serialized here for now.
+ // At present, only serialized(analyzed) define expr is directly used when creating a materialized view.
+ // It will not be used here, but through another structure `TAlterMaterializedViewParam`.
return tColumn;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org