You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/08/28 15:19:04 UTC
kylin git commit: KYLIN-1979 Move hackNoGroupByAggregation to
cube-based storage implementations
Repository: kylin
Updated Branches:
refs/heads/master 46be80550 -> ae9d74790
KYLIN-1979 Move hackNoGroupByAggregation to cube-based storage implementations
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae9d7479
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae9d7479
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae9d7479
Branch: refs/heads/master
Commit: ae9d747904d02f89a4d6616ca201243afbc075cc
Parents: 46be805
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Aug 28 23:19:06 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Aug 28 23:19:06 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 2 +-
.../apache/kylin/cube/RawQueryLastHacker.java | 82 ++++++++++++++++++++
.../org/apache/kylin/measure/MeasureType.java | 2 +-
.../dim/DimCountDistinctMeasureType.java | 10 ++-
.../ExtendedColumnMeasureType.java | 26 ++++---
.../kylin/measure/raw/RawMeasureType.java | 47 +++++------
.../kylin/measure/topn/TopNMeasureType.java | 38 ++++-----
.../kylin/metadata/realization/SQLDigest.java | 5 +-
.../gtrecord/GTCubeStorageQueryBase.java | 23 +++++-
.../kylin/query/enumerator/OLAPEnumerator.java | 63 ---------------
.../storage/hbase/cube/v1/CubeStorageQuery.java | 23 +++++-
11 files changed, 195 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e0d8dd3..79d1e3b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -88,7 +88,7 @@ public class CubeCapabilityChecker {
return result;
}
- if (digest.isRawQuery() && cube.getFactTable().equals(digest.factTable)) {
+ if (digest.isRawQuery && cube.getFactTable().equals(digest.factTable)) {
result.influences.add(new CapabilityInfluence() {
@Override
public double suggestCostMultiplier() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
new file mode 100644
index 0000000..63ddac5
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube;
+
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RawQueryLastHacker {
+
+ private static final Logger logger = LoggerFactory.getLogger(RawQueryLastHacker.class);
+
+ public static void hackNoAggregations(SQLDigest sqlDigest, CubeDesc cubeDesc) {
+ if (!sqlDigest.isRawQuery) {
+ return;
+ }
+
+ // If no group by and metric found, then it's simple query like select ... from ... where ...,
+ // But we have no raw data stored, in order to return better results, we hack to output sum of metric column
+ logger.info("No group by and aggregation found in this query, will hack some result for better look of output...");
+
+ // If it's select * from ...,
+ // We need to retrieve cube to manually add columns into sqlDigest, so that we have full-columns results as output.
+ boolean isSelectAll = sqlDigest.allColumns.isEmpty() || sqlDigest.allColumns.equals(sqlDigest.filterColumns);
+ for (TblColRef col : cubeDesc.listAllColumns()) {
+ if (col.getTable().equals(sqlDigest.factTable) && (cubeDesc.listDimensionColumnsIncludingDerived().contains(col) || isSelectAll)) {
+ sqlDigest.allColumns.add(col);
+ }
+ }
+
+ for (TblColRef col : sqlDigest.allColumns) {
+ if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col)) {
+ // For dimension columns, take them as group by columns.
+ sqlDigest.groupbyColumns.add(col);
+ } else {
+ // For measure columns, take them as metric columns with aggregation function SUM().
+ ParameterDesc colParameter = new ParameterDesc();
+ colParameter.setType("column");
+ colParameter.setValue(col.getName());
+ FunctionDesc sumFunc = new FunctionDesc();
+ sumFunc.setExpression("SUM");
+ sumFunc.setParameter(colParameter);
+
+ boolean measureHasSum = false;
+ for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) {
+ if (colMeasureDesc.getFunction().equals(sumFunc)) {
+ measureHasSum = true;
+ break;
+ }
+ }
+ if (measureHasSum) {
+ sqlDigest.aggregations.add(sumFunc);
+ } else {
+ logger.warn("SUM is not defined for measure column " + col + ", output will be meaningless.");
+ }
+
+ sqlDigest.metricColumns.add(col);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 6786f4b..82618e9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -124,7 +124,7 @@ abstract public class MeasureType<T> {
* They need to adjust dimensions and measures in <code>sqlDigest</code> before scanning,
* such that correct cuboid and measures can be selected by storage.
*/
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
}
/** Return true if one storage record maps to multiple tuples, or false otherwise. */
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
index 7e4e529..9fe1075 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
@@ -18,6 +18,8 @@
package org.apache.kylin.measure.dim;
+import java.util.List;
+
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
@@ -79,8 +81,10 @@ public class DimCountDistinctMeasureType extends MeasureType<Object> {
return DimCountDistinctAggFunc.class;
}
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- sqlDigest.groupbyColumns.addAll(measureDesc.getFunction().getParameter().getColRefs());
- sqlDigest.aggregations.remove(measureDesc.getFunction());
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+ for (MeasureDesc measureDesc : measureDescs) {
+ sqlDigest.groupbyColumns.addAll(measureDesc.getFunction().getParameter().getColRefs());
+ sqlDigest.aggregations.remove(measureDesc.getFunction());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
index dc718d1..796f1f7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
@@ -93,19 +93,21 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
}
@Override
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- FunctionDesc extendColumnFunc = measureDesc.getFunction();
- List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
- TblColRef extended = getExtendedColumn(extendColumnFunc);
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+ for (MeasureDesc measureDesc : measureDescs) {
+ FunctionDesc extendColumnFunc = measureDesc.getFunction();
+ List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
+ TblColRef extended = getExtendedColumn(extendColumnFunc);
+
+ if (!sqlDigest.groupbyColumns.contains(extended)) {
+ return;
+ }
- if (!sqlDigest.groupbyColumns.contains(extended)) {
- return;
+ sqlDigest.aggregations.add(extendColumnFunc);
+ sqlDigest.groupbyColumns.remove(extended);
+ sqlDigest.groupbyColumns.addAll(hosts);
+ sqlDigest.metricColumns.add(extended);
}
-
- sqlDigest.aggregations.add(extendColumnFunc);
- sqlDigest.groupbyColumns.remove(extended);
- sqlDigest.groupbyColumns.addAll(hosts);
- sqlDigest.metricColumns.add(extended);
}
@Override
@@ -151,7 +153,7 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
value = null;
return;
}
-
+
ByteArray byteArray = (ByteArray) measureValue;
//the array in ByteArray is guaranteed to be completed owned by the ByteArray
value = Bytes.toString(byteArray.array());
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index aa46149..50715ec 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -166,7 +166,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc measureDesc) {
//is raw query
- if (!digest.isRawQuery())
+ if (!digest.isRawQuery)
return null;
TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
@@ -196,27 +196,30 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
}
@Override
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- if (sqlDigest.isRawQuery()) {
- TblColRef col = this.getRawColumn(measureDesc.getFunction());
- ParameterDesc colParameter = new ParameterDesc();
- colParameter.setType("column");
- colParameter.setValue(col.getName());
- FunctionDesc rawFunc = new FunctionDesc();
- rawFunc.setExpression("RAW");
- rawFunc.setParameter(colParameter);
-
- if (sqlDigest.allColumns.contains(col)) {
- if (measureDesc.getFunction().equals(rawFunc)) {
- FunctionDesc sumFunc = new FunctionDesc();
- sumFunc.setExpression("SUM");
- sumFunc.setParameter(colParameter);
- sqlDigest.aggregations.remove(sumFunc);
- sqlDigest.aggregations.add(rawFunc);
- logger.info("Add RAW measure on column " + col);
- }
- if (!sqlDigest.metricColumns.contains(col)) {
- sqlDigest.metricColumns.add(col);
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+
+ if (sqlDigest.isRawQuery) {
+ for (MeasureDesc measureDesc : measureDescs) {
+ TblColRef col = this.getRawColumn(measureDesc.getFunction());
+ ParameterDesc colParameter = new ParameterDesc();
+ colParameter.setType("column");
+ colParameter.setValue(col.getName());
+ FunctionDesc rawFunc = new FunctionDesc();
+ rawFunc.setExpression("RAW");
+ rawFunc.setParameter(colParameter);
+
+ if (sqlDigest.allColumns.contains(col)) {
+ if (measureDesc.getFunction().equals(rawFunc)) {
+ FunctionDesc sumFunc = new FunctionDesc();
+ sumFunc.setExpression("SUM");
+ sumFunc.setParameter(colParameter);
+ sqlDigest.aggregations.remove(sumFunc);
+ sqlDigest.aggregations.add(rawFunc);
+ logger.info("Add RAW measure on column " + col);
+ }
+ if (!sqlDigest.metricColumns.contains(col)) {
+ sqlDigest.metricColumns.add(col);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index ad42824..ed22d61 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -39,8 +39,8 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
import org.slf4j.Logger;
@@ -290,28 +290,30 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
}
@Override
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- FunctionDesc topnFunc = measureDesc.getFunction();
- List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc);
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+ for (MeasureDesc measureDesc : measureDescs) {
+ FunctionDesc topnFunc = measureDesc.getFunction();
+ List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc);
- if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false)
- return;
+ if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false)
+ return;
- if (sqlDigest.aggregations.size() > 1) {
- throw new IllegalStateException("When query with topN, only one metrics is allowed.");
- }
+ if (sqlDigest.aggregations.size() > 1) {
+ throw new IllegalStateException("When query with topN, only one metrics is allowed.");
+ }
- if (sqlDigest.aggregations.size() > 0) {
- FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
- if (origFunc.isSum() == false && origFunc.isCount() == false) {
- throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ if (sqlDigest.aggregations.size() > 0) {
+ FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
+ if (origFunc.isSum() == false && origFunc.isCount() == false) {
+ throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ }
+ logger.info("Rewrite function " + origFunc + " to " + topnFunc);
}
- logger.info("Rewrite function " + origFunc + " to " + topnFunc);
- }
- sqlDigest.aggregations = Lists.newArrayList(topnFunc);
- sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
- sqlDigest.metricColumns.addAll(topnLiteralCol);
+ sqlDigest.aggregations = Lists.newArrayList(topnFunc);
+ sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
+ sqlDigest.metricColumns.addAll(topnLiteralCol);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 1eee1e7..d2bba66 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -44,7 +44,9 @@ public class SQLDigest {
public Collection<FunctionDesc> aggregations;
public Collection<MeasureDesc> sortMeasures;
public Collection<OrderEnum> sortOrders;
+ public boolean isRawQuery;
+ //initialized when org.apache.kylin.query.routing.QueryRouter.selectRealization()
public SQLDigest(String factTable, TupleFilter filter, Collection<JoinDesc> joinDescs, Collection<TblColRef> allColumns, //
Collection<TblColRef> groupbyColumns, Collection<TblColRef> filterColumns, Collection<TblColRef> aggregatedColumns, Collection<FunctionDesc> aggregateFunnc, Collection<MeasureDesc> sortMeasures, Collection<OrderEnum> sortOrders) {
this.factTable = factTable;
@@ -57,9 +59,10 @@ public class SQLDigest {
this.aggregations = aggregateFunnc;
this.sortMeasures = sortMeasures;
this.sortOrders = sortOrders;
+ this.isRawQuery = isRawQuery();
}
- public boolean isRawQuery() {
+ private boolean isRawQuery() {
return this.groupbyColumns.isEmpty() && // select a group by a -> not raw
this.aggregations.isEmpty(); // has aggr -> not raw
//the reason to choose aggregations rather than metricColumns is because the former is set earlier at implOLAP
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index bacd293..86346f8 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -22,12 +22,14 @@ import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
@@ -53,6 +55,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@@ -69,7 +72,11 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- // allow custom measures hack
+
+ //cope with queries with no aggregations
+ RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
+
+ // Customized measure taking effect: e.g. allow custom measures to help raw queries
notifyBeforeStorageQuery(sqlDigest);
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
@@ -418,9 +425,21 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
+ Map<String, List<MeasureDesc>> map = Maps.newHashMap();
for (MeasureDesc measure : cubeDesc.getMeasures()) {
MeasureType<?> measureType = measure.getFunction().getMeasureType();
- measureType.adjustSqlDigest(measure, sqlDigest);
+
+ String key = measureType.getClass().getCanonicalName();
+ List<MeasureDesc> temp = null;
+ if ((temp = map.get(key)) != null) {
+ temp.add(measure);
+ } else {
+ map.put(key, Lists.<MeasureDesc> newArrayList(measure));
+ }
+ }
+
+ for (List<MeasureDesc> sublist : map.values()) {
+ sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 96e75f3..dafbb5f 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -25,14 +25,8 @@ import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
@@ -114,7 +108,6 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
// cube don't have correct result for simple query without group by, but let's try to return something makes sense
olapContext.resetSQLDigest();
SQLDigest sqlDigest = olapContext.getSQLDigest();
- hackNoGroupByAggregation(sqlDigest);
// query storage engine
IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
@@ -161,60 +154,4 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
olapContext.storageContext.setThreshold(threshold);
}
- // Hack no-group-by query for better results
- private void hackNoGroupByAggregation(SQLDigest sqlDigest) {
- if (!(olapContext.realization instanceof CubeInstance)) {
- //the hack only makes sense for cubes
- return;
- }
-
- if (!sqlDigest.isRawQuery()) {
- return;
- }
-
- // If no group by and metric found, then it's simple query like select ... from ... where ...,
- // But we have no raw data stored, in order to return better results, we hack to output sum of metric column
- logger.info("No group by and aggregation found in this query, will hack some result for better look of output...");
-
- // If it's select * from ...,
- // We need to retrieve cube to manually add columns into sqlDigest, so that we have full-columns results as output.
- IRealization cube = olapContext.realization;
- boolean isSelectAll = sqlDigest.allColumns.isEmpty() || sqlDigest.allColumns.equals(sqlDigest.filterColumns);
- for (TblColRef col : cube.getAllColumns()) {
- if (col.getTable().equals(sqlDigest.factTable) && (cube.getAllDimensions().contains(col) || isSelectAll)) {
- sqlDigest.allColumns.add(col);
- }
- }
-
- for (TblColRef col : sqlDigest.allColumns) {
- if (cube.getAllDimensions().contains(col)) {
- // For dimension columns, take them as group by columns.
- sqlDigest.groupbyColumns.add(col);
- } else {
- // For measure columns, take them as metric columns with aggregation function SUM().
- ParameterDesc colParameter = new ParameterDesc();
- colParameter.setType("column");
- colParameter.setValue(col.getName());
- FunctionDesc sumFunc = new FunctionDesc();
- sumFunc.setExpression("SUM");
- sumFunc.setParameter(colParameter);
-
- boolean measureHasSum = false;
- for (MeasureDesc colMeasureDesc : cube.getMeasures()) {
- if (colMeasureDesc.getFunction().equals(sumFunc)) {
- measureHasSum = true;
- break;
- }
- }
- if (measureHasSum) {
- sqlDigest.aggregations.add(sumFunc);
- } else {
- logger.warn("SUM is not defined for measure column " + col + ", output will be meaningless.");
- }
-
- sqlDigest.metricColumns.add(col);
- }
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 46f16fe..ff729f4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -42,13 +42,14 @@ import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
import org.apache.kylin.cube.model.RowKeyDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
@@ -97,7 +98,10 @@ public class CubeStorageQuery implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- // allow custom measures hack
+ //cope with queries with no aggregations
+ RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
+
+ // Customized measure taking effect: e.g. allow custom measures to help raw queries
notifyBeforeStorageQuery(sqlDigest);
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
@@ -762,9 +766,22 @@ public class CubeStorageQuery implements IStorageQuery {
}
private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
+
+ Map<String, List<MeasureDesc>> map = Maps.newHashMap();
for (MeasureDesc measure : cubeDesc.getMeasures()) {
MeasureType<?> measureType = measure.getFunction().getMeasureType();
- measureType.adjustSqlDigest(measure, sqlDigest);
+
+ String key = measureType.getClass().getCanonicalName();
+ List<MeasureDesc> temp = null;
+ if ((temp = map.get(key)) != null) {
+ temp.add(measure);
+ } else {
+ map.put(key, Lists.<MeasureDesc> newArrayList(measure));
+ }
+ }
+
+ for (List<MeasureDesc> sublist : map.values()) {
+ sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
}
}