You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/05/18 09:53:14 UTC
[kylin] 04/08: KYLIN-3360 correct count(column)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch KYLIN-3359
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 86719d236366fda43ac23556a010e4fab8275f3b
Author: Zhong <nj...@apache.org>
AuthorDate: Thu May 10 10:00:55 2018 +0800
KYLIN-3360 correct count(column)
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../apache/calcite/sql2rel/SqlToRelConverter.java | 2 +-
.../apache/kylin/query/relnode/ColumnRowType.java | 11 +++++
.../kylin/query/relnode/OLAPAggregateRel.java | 52 ++++++++++++++++++++--
.../apache/kylin/query/relnode/OLAPContext.java | 35 +++++++++++++++
4 files changed, 96 insertions(+), 4 deletions(-)
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index fb5bb19..519a73b 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -4996,7 +4996,7 @@ public class SqlToRelConverter {
// special case for COUNT(*): delete the *
if (operand instanceof SqlIdentifier) {
SqlIdentifier id = (SqlIdentifier) operand;
- if (id.isStar() || isSimpleCount(call)) { /* OVERRIDE POINT */
+ if (id.isStar()) {
assert call.operandCount() == 1;
assert args.isEmpty();
break;
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
index 778d681..65a00e6 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/ColumnRowType.java
@@ -20,6 +20,7 @@ package org.apache.kylin.query.relnode;
import java.util.List;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.expression.ColumnTupleExpression;
import org.apache.kylin.metadata.expression.NoneTupleExpression;
import org.apache.kylin.metadata.expression.TupleExpression;
@@ -70,6 +71,16 @@ public class ColumnRowType {
return -1;
}
+ public Pair<TblColRef, TupleExpression> replaceColumnByIndex(int index, TblColRef newColumn,
+ TupleExpression newTupleExpr) {
+ if (index < 0 || index >= columns.size()) {
+ return null;
+ }
+ TblColRef oldCol = columns.set(index, newColumn);
+ TupleExpression oldExpr = sourceColumns.set(index, newTupleExpr);
+ return new Pair<>(oldCol, oldExpr);
+ }
+
public TupleExpression getSourceColumnsByIndex(int i) {
TupleExpression result = null;
if (sourceColumns != null) {
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 2444daa..0eff905 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -47,6 +47,7 @@ import org.apache.calcite.schema.FunctionParameter;
import org.apache.calcite.schema.impl.AggregateFunctionImpl;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.fun.SqlCountAggFunction;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.fun.SqlSumAggFunction;
import org.apache.calcite.sql.fun.SqlSumEmptyIsZeroAggFunction;
@@ -58,13 +59,18 @@ import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Util;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.measure.ParamAsMeasureCount;
+import org.apache.kylin.metadata.expression.CaseTupleExpression;
import org.apache.kylin.metadata.expression.ColumnTupleExpression;
import org.apache.kylin.metadata.expression.ExpressionColCollector;
import org.apache.kylin.metadata.expression.ExpressionCountDistributor;
import org.apache.kylin.metadata.expression.NumberTupleExpression;
import org.apache.kylin.metadata.expression.TupleExpression;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.DynamicFunctionDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -130,6 +136,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
OLAPContext context;
ColumnRowType columnRowType;
private boolean afterAggregate;
+ private Map<Integer, AggregateCall> hackAggCalls;
private List<AggregateCall> rewriteAggCalls;
private List<TblColRef> groups;
private List<FunctionDesc> aggregations;
@@ -271,7 +278,9 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
void buildAggregations() {
ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
this.aggregations = Lists.newArrayList();
- for (AggregateCall aggCall : this.rewriteAggCalls) {
+ this.hackAggCalls = Maps.newHashMap();
+ for (int i = 0; i < this.rewriteAggCalls.size(); i++) {
+ AggregateCall aggCall = this.rewriteAggCalls.get(i);
ParameterDesc parameter = null;
List<Integer> argList = aggCall.getArgList();
// By default all args are included, UDFs can define their own in getParamAsMeasureCount method.
@@ -309,6 +318,30 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
this.aggregations.add(sumDynFunc);
continue;
}
+ } else if (aggCall.getAggregation() instanceof SqlCountAggFunction && !aggCall.isDistinct()) {
+ if (tupleExpr instanceof ColumnTupleExpression) {
+ TblColRef srcCol = ((ColumnTupleExpression) tupleExpr).getColumn();
+ if (this.context.belongToFactTable(srcCol)) {
+ tupleExpr = getCountColumnExpression(srcCol);
+
+ TblColRef column = TblColRef.newInnerColumn(tupleExpr.getDigest(),
+ TblColRef.InnerDataTypeEnum.LITERAL);
+
+ SumDynamicFunctionDesc sumDynFunc = new SumDynamicFunctionDesc(
+ ParameterDesc.newInstance(column), tupleExpr);
+
+ inputColumnRowType.replaceColumnByIndex(iRowIdx, column, tupleExpr);
+
+ AggregateCall newAggCall = AggregateCall.create(SqlStdOperatorTable.SUM, false,
+ aggCall.getArgList(), -1, aggCall.getType(), aggCall.getName());
+ this.hackAggCalls.put(i, newAggCall);
+
+ this.context.dynamicFields.put(column, aggCall.getType());
+
+ this.aggregations.add(sumDynFunc);
+ continue;
+ }
+ }
}
}
String expression = getAggrFuncName(aggCall);
@@ -337,9 +370,10 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
// only rewrite the innermost aggregation
if (needRewrite()) {
// rewrite the aggCalls
- this.rewriteAggCalls = new ArrayList<AggregateCall>(aggCalls.size());
+ this.rewriteAggCalls = Lists.newArrayListWithExpectedSize(aggCalls.size());
for (int i = 0; i < this.aggCalls.size(); i++) {
- AggregateCall aggCall = this.aggCalls.get(i);
+ AggregateCall aggCall = this.hackAggCalls.get(i) != null ? this.hackAggCalls.get(i)
+ : this.aggCalls.get(i);
FunctionDesc cubeFunc = this.context.aggregations.get(i);
// filter needn,t rewrite aggfunc
// if it's not a cube, then the "needRewriteField func" should not resort to any rewrite fields,
@@ -568,4 +602,16 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
return super.explainTerms(pw).item("ctx",
context == null ? "" : String.valueOf(context.id) + "@" + context.realization);
}
+
+ private TupleExpression getCountColumnExpression(TblColRef colRef) {
+ List<Pair<TupleFilter, TupleExpression>> whenList = Lists.newArrayListWithExpectedSize(1);
+ TupleFilter whenFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.ISNULL);
+ whenFilter.addChild(new ColumnTupleFilter(colRef));
+ whenList.add(new Pair<TupleFilter, TupleExpression>(whenFilter, new NumberTupleExpression(0)));
+
+ TupleExpression elseExpr = new ColumnTupleExpression(SumDynamicFunctionDesc.mockCntCol);
+ TupleExpression ret = new CaseTupleExpression(whenList, elseExpr);
+ ret.setDigest("_KY_COUNT(" + colRef.getName() + ")");
+ return ret;
+ }
}
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 1132cd4..20533ad 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -41,7 +41,9 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinsTree;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.realization.SQLDigest.SQLCall;
@@ -52,6 +54,7 @@ import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hybrid.HybridInstance;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
*/
@@ -208,6 +211,38 @@ public class OLAPContext {
return false;
}
+ public boolean belongToFactTable(TblColRef tblColRef) {
+ if (!belongToContextTables(tblColRef)) {
+ return false;
+ }
+ KylinConfig kylinConfig = olapSchema.getConfig();
+ String projectName = olapSchema.getProjectName();
+ String factTableName = firstTableScan.getOlapTable().getTableName();
+ Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName,
+ factTableName);
+ for (IRealization real : realizations) {
+ DataModelDesc model = real.getModel();
+ TblColRef.fixUnknownModel(model, tblColRef.getTableRef().getTableIdentity(), tblColRef);
+
+ // cannot be a measure column
+ Set<String> metrics = Sets.newHashSet(model.getMetrics());
+ if (metrics.contains(tblColRef.getIdentity())) {
+ tblColRef.unfixTableRef();
+ return false;
+ }
+
+ // must belong to a fact table
+ for (TableRef factTable : model.getFactTables()) {
+ if (factTable.getColumns().contains(tblColRef)) {
+ tblColRef.unfixTableRef();
+ return true;
+ }
+ }
+ tblColRef.unfixTableRef();
+ }
+ return false;
+ }
+
public void setReturnTupleInfo(RelDataType rowType, ColumnRowType columnRowType) {
TupleInfo info = new TupleInfo();
List<RelDataTypeField> fieldList = rowType.getFieldList();
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.