You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/08/01 09:43:36 UTC
[incubator-doris] branch master updated: [MaterializedView] Change
type of sum, min, max function column in mv (#4199)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 25f3420 [MaterializedView] Change type of sum, min, max function column in mv (#4199)
25f3420 is described below
commit 25f3420855652d6de9e70bc9b09c7d1c8ee21b5f
Author: EmmyMiao87 <52...@qq.com>
AuthorDate: Sat Aug 1 17:43:23 2020 +0800
[MaterializedView] Change type of sum, min, max function column in mv (#4199)
If the agg function is sum, the type of mv column will be bigint.
The only exception is that if the base column is largeint, the type of mv column will be largeint.
If the agg function is min or max, the type of mv column will be same as the type of base column.
For example, the type of mv column is smallint when the agg function is min.
---
.../doris/analysis/CreateMaterializedViewStmt.java | 52 +++++++++------
.../org/apache/doris/analysis/MVColumnItem.java | 1 +
.../analysis/CreateMaterializedViewStmtTest.java | 73 +++++++++++++++++-----
3 files changed, 89 insertions(+), 37 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index 35a00e7..f9b26f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -47,10 +47,10 @@ import java.util.Set;
* <p>
* Syntax:
* CREATE MATERIALIZED VIEW [MV name] (
- * SELECT select_expr[, select_expr ...]
- * FROM [Base view name]
- * GROUP BY column_name[, column_name ...]
- * ORDER BY column_name[, column_name ...])
+ * SELECT select_expr[, select_expr ...]
+ * FROM [Base view name]
+ * GROUP BY column_name[, column_name ...]
+ * ORDER BY column_name[, column_name ...])
* [PROPERTIES ("key" = "value")]
*/
public class CreateMaterializedViewStmt extends DdlStmt {
@@ -60,11 +60,11 @@ public class CreateMaterializedViewStmt extends DdlStmt {
static {
FN_NAME_TO_PATTERN = Maps.newHashMap();
FN_NAME_TO_PATTERN.put(AggregateType.SUM.name().toLowerCase(),
- new MVColumnOneChildPattern(AggregateType.SUM.name().toLowerCase()));
+ new MVColumnOneChildPattern(AggregateType.SUM.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(AggregateType.MIN.name().toLowerCase(),
- new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase()));
+ new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(AggregateType.MAX.name().toLowerCase(),
- new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase()));
+ new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(FunctionSet.COUNT, new MVColumnOneChildPattern(FunctionSet.COUNT));
FN_NAME_TO_PATTERN.put(FunctionSet.BITMAP_UNION, new MVColumnBitmapUnionPattern());
FN_NAME_TO_PATTERN.put(FunctionSet.HLL_UNION, new MVColumnHLLUnionPattern());
@@ -78,9 +78,9 @@ public class CreateMaterializedViewStmt extends DdlStmt {
/**
* origin stmt: select k1, k2, v1, sum(v2) from base_table group by k1, k2, v1
* mvColumnItemList: [k1: {name: k1, isKey: true, aggType: null, isAggregationTypeImplicit: false},
- * k2: {name: k2, isKey: true, aggType: null, isAggregationTypeImplicit: false},
- * v1: {name: v1, isKey: true, aggType: null, isAggregationTypeImplicit: false},
- * v2: {name: v2, isKey: false, aggType: sum, isAggregationTypeImplicit: false}]
+ * k2: {name: k2, isKey: true, aggType: null, isAggregationTypeImplicit: false},
+ * v1: {name: v1, isKey: true, aggType: null, isAggregationTypeImplicit: false},
+ * v2: {name: v2, isKey: false, aggType: sum, isAggregationTypeImplicit: false}]
* This order of mvColumnItemList is meaningful.
*/
private List<MVColumnItem> mvColumnItemList = Lists.newArrayList();
@@ -139,16 +139,16 @@ public class CreateMaterializedViewStmt extends DdlStmt {
analyzeFromClause();
if (selectStmt.getWhereClause() != null) {
throw new AnalysisException("The where clause is not supported in add materialized view clause, expr:"
- + selectStmt.getWhereClause().toSql());
+ + selectStmt.getWhereClause().toSql());
}
if (selectStmt.getHavingPred() != null) {
throw new AnalysisException("The having clause is not supported in add materialized view clause, expr:"
- + selectStmt.getHavingPred().toSql());
+ + selectStmt.getHavingPred().toSql());
}
analyzeOrderByClause();
if (selectStmt.getLimit() != -1) {
throw new AnalysisException("The limit clause is not supported in add materialized view clause, expr:"
- + " limit " + selectStmt.getLimit());
+ + " limit " + selectStmt.getLimit());
}
}
@@ -174,7 +174,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
Expr selectListItemExpr = selectListItem.getExpr();
if (!(selectListItemExpr instanceof SlotRef) && !(selectListItemExpr instanceof FunctionCallExpr)) {
throw new AnalysisException("The materialized view only support the single column or function expr. "
- + "Error column: " + selectListItemExpr.toSql());
+ + "Error column: " + selectListItemExpr.toSql());
}
if (selectListItemExpr instanceof SlotRef) {
if (meetAggregate) {
@@ -245,7 +245,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
List<OrderByElement> orderByElements = selectStmt.getOrderByElements();
if (orderByElements.size() > mvColumnItemList.size()) {
throw new AnalysisException("The number of columns in order clause must be less then " + "the number of "
- + "columns in select clause");
+ + "columns in select clause");
}
if (beginIndexOfAggregation != -1 && (orderByElements.size() != (beginIndexOfAggregation))) {
throw new AnalysisException("The key of columns in mv must be all of group by columns");
@@ -254,13 +254,13 @@ public class CreateMaterializedViewStmt extends DdlStmt {
Expr orderByElement = orderByElements.get(i).getExpr();
if (!(orderByElement instanceof SlotRef)) {
throw new AnalysisException("The column in order clause must be original column without calculation. "
- + "Error column: " + orderByElement.toSql());
+ + "Error column: " + orderByElement.toSql());
}
MVColumnItem mvColumnItem = mvColumnItemList.get(i);
SlotRef slotRef = (SlotRef) orderByElement;
if (!mvColumnItem.getName().equalsIgnoreCase(slotRef.getColumnName())) {
throw new AnalysisException("The order of columns in order by clause must be same as "
- + "the order of columns in select list");
+ + "the order of columns in select list");
}
Preconditions.checkState(mvColumnItem.getAggregationType() == null);
mvColumnItem.setIsKey(true);
@@ -339,7 +339,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
}
}
- public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws AnalysisException {
+ private MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws AnalysisException {
String functionName = functionCallExpr.getFnName().getFunction();
List<SlotRef> slots = new ArrayList<>();
functionCallExpr.collect(SlotRef.class, slots);
@@ -353,11 +353,23 @@ public class CreateMaterializedViewStmt extends DdlStmt {
Type type;
switch (functionName.toLowerCase()) {
case "sum":
+ mvColumnName = baseColumnName;
+ mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
+ PrimitiveType baseColumnType = baseColumnRef.getType().getPrimitiveType();
+ if (baseColumnType == PrimitiveType.TINYINT || baseColumnType == PrimitiveType.SMALLINT
+ || baseColumnType == PrimitiveType.INT) {
+ type = Type.BIGINT;
+ } else if (baseColumnType == PrimitiveType.FLOAT) {
+ type = Type.DOUBLE;
+ } else {
+ type = Type.fromPrimitiveType(baseColumnRef.getType().getPrimitiveType());
+ }
+ break;
case "min":
case "max":
mvColumnName = baseColumnName;
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
- type = Type.BIGINT;
+ type = Type.fromPrimitiveType(baseColumnRef.getType().getPrimitiveType());
break;
case FunctionSet.BITMAP_UNION:
// Compatible aggregation models
@@ -445,7 +457,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
case FunctionSet.COUNT:
Expr defineExpr = new CaseExpr(null, Lists.newArrayList(
new CaseWhenClause(new IsNullPredicate(slots.get(0), false),
- new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT));
+ new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT));
result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr);
break;
default:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java
index 55762cb..37137e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java
@@ -105,6 +105,7 @@ public class MVColumnItem {
} else {
Column result = new Column(baseColumn);
result.setIsKey(isKey);
+ result.setType(type);
result.setAggregationType(aggregationType, isAggregationTypeImplicit);
return result;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java
index da42668..fdfcec5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
@@ -325,7 +326,7 @@ public class CreateMaterializedViewStmtTest {
@Test
public void testDuplicateColumn1(@Injectable SlotRef slotRef1,
- @Injectable SelectStmt selectStmt) throws UserException {
+ @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -415,7 +416,7 @@ public class CreateMaterializedViewStmtTest {
@Injectable SlotRef slotRef4,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt,
- @Injectable AggregateInfo aggregateInfo) throws UserException {
+ @Injectable AggregateInfo aggregateInfo) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -506,8 +507,8 @@ public class CreateMaterializedViewStmtTest {
@Test
public void testMVColumnsWithoutOrderbyWithoutAggregation(@Injectable SlotRef slotRef1,
- @Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
- @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
+ @Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
+ @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -606,8 +607,8 @@ public class CreateMaterializedViewStmtTest {
*/
@Test
public void testMVColumnsWithoutOrderbyWithoutAggregationWithFloat(@Injectable SlotRef slotRef1,
- @Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
- @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
+ @Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
+ @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -704,8 +705,8 @@ public class CreateMaterializedViewStmtTest {
*/
@Test
public void testMVColumnsWithoutOrderbyWithoutAggregationWithVarchar(@Injectable SlotRef slotRef1,
- @Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
- @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
+ @Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
+ @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -802,7 +803,7 @@ public class CreateMaterializedViewStmtTest {
*/
@Test
public void testMVColumnsWithFirstFloat(@Injectable SlotRef slotRef1,
- @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
+ @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -850,7 +851,7 @@ public class CreateMaterializedViewStmtTest {
*/
@Test
public void testMVColumnsWithFirstVarchar(@Injectable SlotRef slotRef1,
- @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
+ @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -903,10 +904,10 @@ public class CreateMaterializedViewStmtTest {
@Test
public void testMVColumns(@Injectable SlotRef slotRef1,
- @Injectable SlotRef slotRef2,
- @Injectable TableRef tableRef,
- @Injectable SelectStmt selectStmt,
- @Injectable AggregateInfo aggregateInfo) throws UserException {
+ @Injectable SlotRef slotRef2,
+ @Injectable TableRef tableRef,
+ @Injectable SelectStmt selectStmt,
+ @Injectable AggregateInfo aggregateInfo) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -981,9 +982,9 @@ public class CreateMaterializedViewStmtTest {
@Test
public void testDeduplicateMV(@Injectable SlotRef slotRef1,
- @Injectable TableRef tableRef,
- @Injectable SelectStmt selectStmt,
- @Injectable AggregateInfo aggregateInfo) throws UserException {
+ @Injectable TableRef tableRef,
+ @Injectable SelectStmt selectStmt,
+ @Injectable AggregateInfo aggregateInfo) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
@@ -1022,5 +1023,43 @@ public class CreateMaterializedViewStmtTest {
}
}
+
+ @Test
+ public void testBuildMVColumnItem(@Injectable SelectStmt selectStmt) {
+ CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
+ SlotRef slotRef = new SlotRef(new TableName("db", "table"), "a");
+ List<Expr> params = Lists.newArrayList();
+ params.add(slotRef);
+ FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", params);
+ slotRef.setType(Type.LARGEINT);
+ MVColumnItem mvColumnItem = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", functionCallExpr);
+ Assert.assertEquals(Type.LARGEINT, mvColumnItem.getType());
+
+ SlotRef slotRef2 = new SlotRef(new TableName("db", "table"), "a");
+ List<Expr> params2 = Lists.newArrayList();
+ params2.add(slotRef2);
+ FunctionCallExpr functionCallExpr2 = new FunctionCallExpr("sum", params2);
+ slotRef2.setType(Type.SMALLINT);
+ MVColumnItem mvColumnItem2 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", functionCallExpr2);
+ Assert.assertEquals(Type.BIGINT, mvColumnItem2.getType());
+
+ SlotRef slotRef3 = new SlotRef(new TableName("db", "table"), "a");
+ List<Expr> params3 = Lists.newArrayList();
+ params3.add(slotRef3);
+ FunctionCallExpr functionCallExpr3 = new FunctionCallExpr("min", params3);
+ slotRef3.setType(Type.VARCHAR);
+ MVColumnItem mvColumnItem3 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", functionCallExpr3);
+ Assert.assertEquals(Type.VARCHAR, mvColumnItem3.getType());
+
+ SlotRef slotRef4 = new SlotRef(new TableName("db", "table"), "a");
+ List<Expr> params4 = Lists.newArrayList();
+ params4.add(slotRef4);
+ FunctionCallExpr functionCallExpr4 = new FunctionCallExpr("sum", params4);
+ slotRef4.setType(Type.DOUBLE);
+ MVColumnItem mvColumnItem4 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", functionCallExpr4);
+ Assert.assertEquals(Type.DOUBLE, mvColumnItem4.getType());
+
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org