You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/31 13:01:57 UTC
[30/50] incubator-kylin git commit: KYLIN-512 fixing min(cal_dt) issue
KYLIN-512 fixing min(cal_dt) issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f866c824
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f866c824
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f866c824
Branch: refs/heads/inverted-index
Commit: f866c824115e76a5ba26b2b7200c780ed6e7b4b4
Parents: bd1efca
Author: honma <ho...@ebay.com>
Authored: Wed Jan 28 15:02:06 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Jan 28 18:51:20 2015 +0800
----------------------------------------------------------------------
.../kylinolap/invertedindex/model/IIDesc.java | 23 ++++++
.../metadata/filter/ExtractTupleFilter.java | 1 -
.../kylinolap/metadata/model/FunctionDesc.java | 2 +-
.../kylinolap/query/routing/RoutingRule.java | 2 +-
.../AdjustForWeeklyMatchCubeRule.java | 67 -----------------
.../AdjustForWeeklyMatchedRealization.java | 78 ++++++++++++++++++++
.../kylinolap/query/test/KylinQueryTest.java | 2 +-
query/src/test/resources/query/sql/query77.sql | 3 +
.../hbase/coprocessor/CoprocessorFilter.java | 4 +-
.../endpoint/EndpointAggregators.java | 4 +-
10 files changed, 113 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java
index 858f8d5..98d58ba 100644
--- a/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java
+++ b/invertedindex/src/main/java/com/kylinolap/invertedindex/model/IIDesc.java
@@ -2,6 +2,7 @@ package com.kylinolap.invertedindex.model;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashSet;
import java.util.List;
@@ -138,6 +139,7 @@ public class IIDesc extends RootPersistentEntity {
measureDescs.add(makeMeasureDescs("MAX", col));
// TODO support for HLL
}
+ measureDescs.add(makeCountMeasure());
// partitioning column
tsCol = -1;
@@ -169,6 +171,14 @@ public class IIDesc extends RootPersistentEntity {
return measureDescs;
}
+ public List<FunctionDesc> listAllFunctions() {
+ List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
+ for (MeasureDesc m : measureDescs) {
+ functions.add(m.getFunction());
+ }
+ return functions;
+ }
+
private MeasureDesc makeMeasureDescs(String func, ColumnDesc columnDesc) {
String columnName = columnDesc.getName();
String returnType = columnDesc.getTypeName();
@@ -185,6 +195,19 @@ public class IIDesc extends RootPersistentEntity {
return measureDesc;
}
+ private MeasureDesc makeCountMeasure() {
+ MeasureDesc measureDesc = new MeasureDesc();
+ FunctionDesc f1 = new FunctionDesc();
+ f1.setExpression("COUNT");
+ ParameterDesc p1 = new ParameterDesc();
+ p1.setType("constant");
+ p1.setValue("1");
+ f1.setParameter(p1);
+ f1.setReturnType("bigint");
+ measureDesc.setFunction(f1);
+ return measureDesc;
+ }
+
/**
* at first stage the only table in II is fact table, tables
*
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java b/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
index 1467ee8..9cfe08d 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/filter/ExtractTupleFilter.java
@@ -54,7 +54,6 @@ public class ExtractTupleFilter extends TupleFilter {
return true;
}
- //askliyang
private int extractDate(String type, int inDate) {
// this shifts the epoch back to astronomical year -4800 instead of the
// start of the Christian era in year AD 1 of the proleptic Gregorian
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/metadata/src/main/java/com/kylinolap/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/FunctionDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/FunctionDesc.java
index f23c184..d42247c 100644
--- a/metadata/src/main/java/com/kylinolap/metadata/model/FunctionDesc.java
+++ b/metadata/src/main/java/com/kylinolap/metadata/model/FunctionDesc.java
@@ -174,7 +174,7 @@ public class FunctionDesc {
final int prime = 31;
int result = 1;
result = prime * result + ((expression == null) ? 0 : expression.hashCode());
- result = prime * result + ((parameter == null) ? 0 : parameter.hashCode());
+ result = prime * result + ((isCount() || parameter == null) ? 0 : parameter.hashCode());
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/query/src/main/java/com/kylinolap/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/routing/RoutingRule.java b/query/src/main/java/com/kylinolap/query/routing/RoutingRule.java
index ded7650..a2e302c 100644
--- a/query/src/main/java/com/kylinolap/query/routing/RoutingRule.java
+++ b/query/src/main/java/com/kylinolap/query/routing/RoutingRule.java
@@ -27,7 +27,7 @@ public abstract class RoutingRule {
rules.add(new RealizationPriorityRule());
rules.add(new SimpleQueryMoreColumsCubeFirstRule());
rules.add(new CubesSortRule());
- rules.add(new AdjustForWeeklyMatchCubeRule());//this rule might modify olapcontext content, better put it at last
+ rules.add(new AdjustForWeeklyMatchedRealization());//this rule might modify olapcontext content, better put it at last
}
public static void applyRules(List<IRealization> realizations, OLAPContext olapContext) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/query/src/main/java/com/kylinolap/query/routing/RoutingRules/AdjustForWeeklyMatchCubeRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/routing/RoutingRules/AdjustForWeeklyMatchCubeRule.java b/query/src/main/java/com/kylinolap/query/routing/RoutingRules/AdjustForWeeklyMatchCubeRule.java
deleted file mode 100644
index ba91789..0000000
--- a/query/src/main/java/com/kylinolap/query/routing/RoutingRules/AdjustForWeeklyMatchCubeRule.java
+++ /dev/null
@@ -1,67 +0,0 @@
-package com.kylinolap.query.routing.RoutingRules;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import com.kylinolap.cube.CubeCapabilityChecker;
-import com.kylinolap.cube.CubeDimensionDeriver;
-import com.kylinolap.cube.CubeInstance;
-import com.kylinolap.cube.model.CubeDesc;
-import com.kylinolap.metadata.model.FunctionDesc;
-import com.kylinolap.metadata.model.TblColRef;
-import com.kylinolap.metadata.realization.IRealization;
-import com.kylinolap.query.relnode.OLAPContext;
-import com.kylinolap.query.routing.RoutingRule;
-import org.eigenbase.reltype.RelDataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 1/5/15.
- */
-public class AdjustForWeeklyMatchCubeRule extends RoutingRule {
- private static final Logger logger = LoggerFactory.getLogger(AdjustForWeeklyMatchCubeRule.class);
-
- @Override
- public void apply(List<IRealization> realizations, OLAPContext olapContext) {
- if (realizations.size() > 0) {
- IRealization first = realizations.get(0);
- if (first instanceof CubeInstance) {
- CubeInstance cube = (CubeInstance) first;
- if (!CubeCapabilityChecker.check(cube, olapContext.getSQLDigest(), false)) {
- adjustOLAPContext(cube, olapContext);
- }
- }
- }
- }
-
- private static void adjustOLAPContext(CubeInstance cube, OLAPContext olapContext) {
- Collection<TblColRef> dimensionColumns = CubeDimensionDeriver.getDimensionColumns(olapContext.groupByColumns, olapContext.filterColumns);
- Collection<FunctionDesc> functions = olapContext.aggregations;
- Collection<TblColRef> metricsColumns = olapContext.metricsColumns;
- Map<String, RelDataType> rewriteFields = olapContext.rewriteFields;
-
- CubeDesc cubeDesc = cube.getDescriptor();
- Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
-
- Iterator<FunctionDesc> it = functions.iterator();
- while (it.hasNext()) {
- FunctionDesc functionDesc = it.next();
- if (!cubeFuncs.contains(functionDesc)) {
- // try to convert the metric to dimension to see if it works
- TblColRef col = functionDesc.selectTblColRef(metricsColumns, cubeDesc.getFactTable());
- functionDesc.setAppliedOnDimension(true);
- rewriteFields.remove(functionDesc.getRewriteFieldName());
- if (col != null) {
- metricsColumns.remove(col);
- dimensionColumns.add(col);
- olapContext.storageContext.addOtherMandatoryColumns(col);
- }
- logger.info("Adjust OLAPContext for " + functionDesc);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/query/src/main/java/com/kylinolap/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java b/query/src/main/java/com/kylinolap/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
new file mode 100644
index 0000000..6f8bd16
--- /dev/null
+++ b/query/src/main/java/com/kylinolap/query/routing/RoutingRules/AdjustForWeeklyMatchedRealization.java
@@ -0,0 +1,78 @@
+package com.kylinolap.query.routing.RoutingRules;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.kylinolap.cube.CubeCapabilityChecker;
+import com.kylinolap.cube.CubeInstance;
+import com.kylinolap.cube.model.CubeDesc;
+import com.kylinolap.invertedindex.IIInstance;
+import com.kylinolap.invertedindex.model.IIDesc;
+import com.kylinolap.metadata.model.FunctionDesc;
+import com.kylinolap.metadata.model.TblColRef;
+import com.kylinolap.metadata.realization.IRealization;
+import com.kylinolap.query.relnode.OLAPContext;
+import com.kylinolap.query.routing.RoutingRule;
+
+/**
+ * Created by Hongbin Ma(Binmahone) on 1/5/15.
+ */
+public class AdjustForWeeklyMatchedRealization extends RoutingRule {
+ private static final Logger logger = LoggerFactory.getLogger(AdjustForWeeklyMatchedRealization.class);
+
+ @Override
+ public void apply(List<IRealization> realizations, OLAPContext olapContext) {
+ if (realizations.size() > 0) {
+ IRealization first = realizations.get(0);
+
+ if (first instanceof CubeInstance) {
+ CubeInstance cube = (CubeInstance) first;
+ adjustOLAPContextIfNecessary(cube, olapContext);
+ }
+
+ if (first instanceof IIInstance) {
+ IIInstance ii = (IIInstance) first;
+ adjustOLAPContextIfNecessary(ii, olapContext);
+ }
+ }
+ }
+
+ private static void adjustOLAPContextIfNecessary(IIInstance ii, OLAPContext olapContext) {
+ IIDesc iiDesc = ii.getDescriptor();
+ Collection<FunctionDesc> iiFuncs = iiDesc.listAllFunctions();
+ convertAggreationToDimension(olapContext, iiFuncs, iiDesc.getFactTableName());
+ }
+
+ private static void adjustOLAPContextIfNecessary(CubeInstance cube, OLAPContext olapContext) {
+ if (CubeCapabilityChecker.check(cube, olapContext.getSQLDigest(), false))
+ return;
+
+ CubeDesc cubeDesc = cube.getDescriptor();
+ Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
+ convertAggreationToDimension(olapContext, cubeFuncs, cubeDesc.getFactTable());
+ }
+
+ private static void convertAggreationToDimension(OLAPContext olapContext, Collection<FunctionDesc> availableAggregations, String factTableName) {
+ Iterator<FunctionDesc> it = olapContext.aggregations.iterator();
+ while (it.hasNext()) {
+ FunctionDesc functionDesc = it.next();
+ if (!availableAggregations.contains(functionDesc)) {
+ // try to convert the metric to dimension to see if it works
+ TblColRef col = functionDesc.selectTblColRef(olapContext.metricsColumns, factTableName);
+ functionDesc.setAppliedOnDimension(true);
+ olapContext.rewriteFields.remove(functionDesc.getRewriteFieldName());
+ if (col != null) {
+ olapContext.metricsColumns.remove(col);
+ olapContext.groupByColumns.add(col);
+ olapContext.storageContext.addOtherMandatoryColumns(col);
+ }
+ logger.info("Adjust OLAPContext for " + functionDesc);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java b/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java
index 5f37c75..29edddc 100644
--- a/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java
+++ b/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java
@@ -133,7 +133,7 @@ public class KylinQueryTest extends KylinTestBase {
@Test
public void testSingleRunQuery() throws Exception {
- String queryFileName = "src/test/resources/query/sql/query58.sql";
+ String queryFileName = "src/test/resources/query/sql/query62.sql";
File sqlFile = new File(queryFileName);
runSQL(sqlFile, true, true);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/query/src/test/resources/query/sql/query77.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query77.sql b/query/src/test/resources/query/sql/query77.sql
new file mode 100644
index 0000000..829d61d
--- /dev/null
+++ b/query/src/test/resources/query/sql/query77.sql
@@ -0,0 +1,3 @@
+select test_kylin_fact.lstg_format_name,sum(test_kylin_fact.price) as GMV
+ , count(*) as TRANS_CNT from test_kylin_fact
+ group by test_kylin_fact.lstg_format_name having sum(price)>5000 and count(*)>72
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java
index 8ad2079..c5359a3 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/CoprocessorFilter.java
@@ -56,7 +56,9 @@ public class CoprocessorFilter {
if (filter == null)
return null;
- //askliyang
+ // In case of NOT(unEvaluatableFilter), we should immediatedly replace it as TRUE,
+ // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter) will
+ // always return FALSE
if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
TupleFilter.collectColumns(filter, unstrictlyFilteredColumns);
return ConstantTupleFilter.TRUE;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f866c824/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointAggregators.java b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
index 30371f4..d3ff956 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/coprocessor/endpoint/EndpointAggregators.java
@@ -50,7 +50,9 @@ public class EndpointAggregators {
dataTypes[i] = functionDesc.getReturnType();
if (functionDesc.isCount()) {
- refColIndex[i] = -1;
+ refColIndex[i] = -1;//-1 for count, -2 for metricOnDimension
+ } else if (functionDesc.isAppliedOnDimension()) {
+ refColIndex[i] = -2;
} else {
refColIndex[i] = tableInfo.findMetric(functionDesc.getParameter().getValue());
if (refColIndex[i] < 0) {