You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/10/20 11:00:58 UTC
incubator-kylin git commit: KYLIN-1075 Gives better (incorrect)
result for simple queries without group-by
Repository: incubator-kylin
Updated Branches:
refs/heads/1.x-staging 88c660fd2 -> b6b22b2e4
KYLIN-1075 Gives better (incorrect) result for simple queries without group-by
Signed-off-by: Li, Yang <ya...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b6b22b2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b6b22b2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b6b22b2e
Branch: refs/heads/1.x-staging
Commit: b6b22b2e47b33ecba5c4ec1381f140413bb6c4c7
Parents: 88c660f
Author: lidongsjtu <li...@126.com>
Authored: Tue Oct 20 16:27:40 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Oct 20 16:27:40 2015 +0800
----------------------------------------------------------------------
.../kylin/metadata/project/ProjectL2Cache.java | 2 +-
.../kylin/query/enumerator/CubeEnumerator.java | 64 +++++++++++++++++++-
2 files changed, 64 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b6b22b2e/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 2277f40..ead0329 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -223,7 +223,7 @@ class ProjectL2Cache {
// auto-define table required by realization for some legacy test case
if (prjCache.tables.get(table.getIdentity()) == null) {
prjCache.tables.put(table.getIdentity(), new TableCache(table));
- logger.warn("Realization '" + realization.getCanonicalName() + "' reports columcn '" + col.getCanonicalName() + "' whose table is not defined in project '" + prjCache.project + "'");
+ logger.warn("Realization '" + realization.getCanonicalName() + "' reports column '" + col.getCanonicalName() + "' whose table is not defined in project '" + prjCache.project + "'");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b6b22b2e/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
index 64a55a0..66a4035 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/CubeEnumerator.java
@@ -28,6 +28,12 @@ import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.query.relnode.OLAPContext;
@@ -134,9 +140,13 @@ public class CubeEnumerator implements Enumerator<Object[]> {
// bind dynamic variables
bindVariable(olapContext.filter);
+ // cube don't have correct result for simple query without group by, but let's try to return something makes sense
+ SQLDigest sqlDigest = olapContext.getSQLDigest();
+ hackNoGroupByAggregation(sqlDigest);
+
// query storage engine
IStorageEngine storageEngine = StorageEngineFactory.getStorageEngine(olapContext.realization);
- ITupleIterator iterator = storageEngine.search(olapContext.storageContext, olapContext.getSQLDigest());
+ ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest);
if (logger.isDebugEnabled()) {
logger.debug("return TupleIterator...");
}
@@ -176,4 +186,56 @@ public class CubeEnumerator implements Enumerator<Object[]> {
int threshold = Integer.valueOf(propThreshold);
olapContext.storageContext.setThreshold(threshold);
}
+
+ // Hack no-group-by query for better results
+ private void hackNoGroupByAggregation(SQLDigest sqlDigest) {
+ if (!sqlDigest.groupbyColumns.isEmpty() || !sqlDigest.metricColumns.isEmpty())
+ return;
+
+ // If no group by and metric found, then it's simple query like select ... from ... where ...,
+ // But we have no raw data stored, in order to return better results, we hack to output sum of metric column
+ logger.info("No group by and aggregation found in this query, will hack some result for better look of output...");
+
+ // If it's select * from ...,
+ // We need to retrieve cube to manually add columns into sqlDigest, so that we have full-columns results as output.
+ IRealization cube = olapContext.realization;
+ boolean isSelectAll = sqlDigest.allColumns.isEmpty() || sqlDigest.allColumns.equals(sqlDigest.filterColumns);
+ for (TblColRef col : cube.getAllColumns()) {
+ if (col.getTable().equals(sqlDigest.factTable) && (cube.getAllDimensions().contains(col) || isSelectAll)) {
+ sqlDigest.allColumns.add(col);
+ }
+ }
+
+ for (TblColRef col : sqlDigest.allColumns) {
+ // For dimension columns, take them as group by columns.
+ if (cube.getAllDimensions().contains(col)) {
+ sqlDigest.groupbyColumns.add(col);
+ }
+ // For measure columns, take them as metric columns with aggregation function SUM().
+ else {
+ ParameterDesc colParameter = new ParameterDesc();
+ colParameter.setType("column");
+ colParameter.setValue(col.getName());
+ FunctionDesc sumFunc = new FunctionDesc();
+ sumFunc.setExpression("SUM");
+ sumFunc.setParameter(colParameter);
+
+ boolean measureHasSum = false;
+ for (MeasureDesc colMeasureDesc : cube.getMeasures()) {
+ if (colMeasureDesc.getFunction().equals(sumFunc)) {
+ measureHasSum = true;
+ break;
+ }
+ }
+ if (measureHasSum) {
+ sqlDigest.aggregations.add(sumFunc);
+ } else {
+ logger.warn("SUM is not defined for measure column " + col + ", output will be meaningless.");
+ }
+
+ sqlDigest.metricColumns.add(col);
+ }
+ }
+ }
+
}