You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/19 00:46:04 UTC
incubator-kylin git commit: KYLIN-780 correct type mistach caused by
agg rewrite
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-780 fcb204467 -> 3e1ea920c
KYLIN-780 correct type mistach caused by agg rewrite
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3e1ea920
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3e1ea920
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3e1ea920
Branch: refs/heads/KYLIN-780
Commit: 3e1ea920cc5bee50f027218bc5643c5f438d979b
Parents: fcb2044
Author: Yang Li <li...@apache.org>
Authored: Fri Jun 19 06:45:40 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Jun 19 06:45:40 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/model/DataType.java | 3 ++-
.../kylin/metadata/model/FunctionDesc.java | 8 ++++----
.../kylin/query/relnode/OLAPAggregateRel.java | 21 +++-----------------
.../apache/kylin/query/schema/OLAPTable.java | 12 +++++------
.../storage/hbase/CubeSegmentTupleIterator.java | 2 +-
.../endpoint/EndpointTupleIterator.java | 2 +-
6 files changed, 17 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3e1ea920/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
index a4e8db6..9f6d38a 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/DataType.java
@@ -32,7 +32,7 @@ import java.util.regex.Pattern;
*
*/
public class DataType {
-
+
public static final String VALID_TYPES_STRING = "any|char|varchar|boolean|binary" //
+ "|integer|tinyint|smallint|bigint|decimal|numeric|float|real|double" //
+ "|date|time|datetime|timestamp|byte|int|short|long|string|hllc" //
@@ -89,6 +89,7 @@ public class DataType {
private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();
+ public static final DataType ANY = DataType.getInstance("any");
public static DataType getInstance(String type) {
if (type == null)
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3e1ea920/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index eda31a1..5f21fff 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -138,13 +138,13 @@ public class FunctionDesc {
this.returnDataType = returnDataType;
}
- public String getSQLType() {
+ public DataType getSQLType() {
if (isCountDistinct())
- return "any";
+ return DataType.ANY;
else if (isSum() || isMax() || isMin())
- return parameter.getColRefs().get(0).getType().getName();
+ return parameter.getColRefs().get(0).getType();
else
- return returnType;
+ return returnDataType;
}
public String getReturnType() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3e1ea920/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 2003bc6..2d22168 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -54,6 +54,7 @@ import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Util;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
@@ -124,13 +125,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel, EnumerableRe
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner) {
return super.computeSelfCost(planner).multiplyBy(.05);
-// double factor = .5;
-// for (AggregateCall aggCall : aggCalls) {
-// if ("$SUM0".equals(aggCall.getAggregation().getName())) {
-// factor = .2;
-// }
-// }
-// return super.computeSelfCost(planner).multiplyBy(factor);
}
@Override
@@ -288,6 +282,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel, EnumerableRe
}
// rebuild rowType & columnRowType
+ ClassUtil.updateFinalField(Aggregate.class, "aggCalls", this, rewriteAggCalls);
this.rowType = this.deriveRowType();
this.columnRowType = this.buildColumnRowType();
@@ -307,22 +302,12 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel, EnumerableRe
if (func.isCountDistinct()) {
newAgg = createHyperLogLogAggFunction(fieldType);
} else if (func.isCount()) {
- //newAgg = new SqlSumEmptyIsZeroAggFunction(fieldType);
newAgg = SqlStdOperatorTable.SUM0;
}
// rebuild aggregate call
AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, newAgg.getName());
-
- // To make sure specified type matches the inferReturnType, or otherwise
- // there will be assertion failure in optiq
- // The problem is BIGINT != BIGINT NOT NULL
- // Details see https://github.scm.corp.ebay.com/Kylin/Kylin/issues/323
- SqlAggFunction aggFunction = (SqlAggFunction) newAggCall.getAggregation();
- AggCallBinding callBinding = newAggCall.createBinding(this);
- RelDataType inferReturnType = aggFunction.inferReturnType(callBinding);
-
- return new AggregateCall(newAgg, false, newArgList, inferReturnType, newAgg.getName());
+ return newAggCall;
}
private SqlAggFunction createHyperLogLogAggFunction(RelDataType returnType) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3e1ea920/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 42b4058..f4eaa08 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -118,12 +118,12 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
if (this.rowType == null) {
// always build exposedColumns and rowType together
this.exposedColumns = listSourceColumns();
- this.rowType = deduceRowType(typeFactory);
+ this.rowType = deriveRowType(typeFactory);
}
return this.rowType;
}
- private RelDataType deduceRowType(RelDataTypeFactory typeFactory) {
+ private RelDataType deriveRowType(RelDataTypeFactory typeFactory) {
RelDataTypeFactory.FieldInfoBuilder fieldInfo = typeFactory.builder();
for (ColumnDesc column : exposedColumns) {
RelDataType sqlType = createSqlType(typeFactory, column);
@@ -149,8 +149,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
else
result = typeFactory.createSqlType(sqlTypeName);
- // due to left join and uncertain data quality, dimension value can be
- // null
+ // due to left join and uncertain data quality, dimension value can be null
if (column.isNullable()) {
result = typeFactory.createTypeWithNullability(result, true);
} else {
@@ -173,8 +172,9 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
metFields.add(fieldName);
ColumnDesc fakeCountCol = new ColumnDesc();
fakeCountCol.setName(fieldName);
- fakeCountCol.setDatatype(func.getSQLType());
- fakeCountCol.setNullable(false);
+ fakeCountCol.setDatatype(func.getSQLType().toString());
+ if (func.isCount())
+ fakeCountCol.setNullable(false);
fakeCountCol.init(sourceTable);
exposedColumns.add(fakeCountCol);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3e1ea920/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
index 4589e4d..268c098 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeSegmentTupleIterator.java
@@ -315,7 +315,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
List<String> names = rowValueDecoder.getNames();
MeasureDesc[] measures = rowValueDecoder.getMeasures();
for (int i = 0; i < measures.length; i++) {
- String dataType = measures[i].getFunction().getSQLType();
+ String dataType = measures[i].getFunction().getSQLType().getName();
info.setField(names.get(i), null, dataType, index++);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3e1ea920/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
index 8b4d8cf..0071307 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/EndpointTupleIterator.java
@@ -241,7 +241,7 @@ public class EndpointTupleIterator implements ITupleIterator {
}
for (FunctionDesc measure : measures) {
- info.setField(measure.getRewriteFieldName(), null, measure.getSQLType(), index++);
+ info.setField(measure.getRewriteFieldName(), null, measure.getSQLType().getName(), index++);
}
return info;