You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/05/22 11:45:59 UTC
[1/2] kylin git commit: KYLIN-2635 optimize determined case when
filters
Repository: kylin
Updated Branches:
refs/heads/2.0.x 9ae792c51 -> 94cd373fb
KYLIN-2635 optimize determined case when filters
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d794046
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d794046
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d794046
Branch: refs/heads/2.0.x
Commit: 5d7940469d0340f1c1126c099b011c83c77535dc
Parents: 9ae792c
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon May 22 15:05:20 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon May 22 19:42:08 2017 +0800
----------------------------------------------------------------------
.../kylin/dict/BuiltInFunctionTransformer.java | 2 +
.../kylin/metadata/filter/CaseTupleFilter.java | 52 ++++++++++--
.../metadata/filter/CompareTupleFilter.java | 29 +++++--
.../filter/FilterOptimizeTransformer.java | 88 ++++++++++++++------
.../filter/IOptimizeableTupleFilter.java | 22 +++++
.../filter/ITupleFilterTransformer.java | 5 ++
.../metadata/filter/LogicalTupleFilter.java | 27 +++++-
.../kylin/metadata/filter/TupleFilter.java | 1 -
.../gtrecord/GTCubeStorageQueryBase.java | 26 +++---
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 1 +
.../kylin/rest/controller/UserController.java | 4 +-
12 files changed, 203 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 378efdd..9d34fbb 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -41,6 +41,8 @@ import com.google.common.primitives.Primitives;
/**
* only take effect when the compare filter has function
+ *
+ * is a first type transformer defined in ITupleFilterTransformer
*/
public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
public static final Logger logger = LoggerFactory.getLogger(BuiltInFunctionTransformer.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
index 2b00d69..9083212 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CaseTupleFilter.java
@@ -26,11 +26,13 @@ import java.util.List;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import com.google.common.collect.Lists;
+
/**
* @author xjiang
*
*/
-public class CaseTupleFilter extends TupleFilter {
+public class CaseTupleFilter extends TupleFilter implements IOptimizeableTupleFilter {
private List<TupleFilter> whenFilters;
private List<TupleFilter> thenFilters;
@@ -40,6 +42,12 @@ public class CaseTupleFilter extends TupleFilter {
public CaseTupleFilter() {
super(new ArrayList<TupleFilter>(), FilterOperatorEnum.CASE);
+ reinit();
+ }
+
+ private void reinit() {
+ this.children.clear();
+
this.filterIndex = 0;
this.values = Collections.emptyList();
this.whenFilters = new ArrayList<TupleFilter>();
@@ -47,14 +55,31 @@ public class CaseTupleFilter extends TupleFilter {
this.elseFilter = null;
}
+ public List<TupleFilter> getWhenFilters() {
+ return Collections.unmodifiableList(whenFilters);
+ }
+
+ public List<TupleFilter> getThenFilters() {
+ return Collections.unmodifiableList(thenFilters);
+ }
+
+ public TupleFilter getElseFilter() {
+ return elseFilter;
+ }
+
@Override
public void addChild(TupleFilter child) {
- super.addChild(child);
+
if (this.filterIndex % 2 == 0) {
- this.whenFilters.add(child);
+ this.elseFilter = child;
} else {
+ this.whenFilters.add(this.elseFilter);
this.thenFilters.add(child);
+ this.elseFilter = null;
}
+
+ super.addChild(child);
+
this.filterIndex++;
}
@@ -65,9 +90,7 @@ public class CaseTupleFilter extends TupleFilter {
@Override
public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
- if (whenFilters.size() != thenFilters.size()) {
- elseFilter = whenFilters.remove(whenFilters.size() - 1);
- }
+
boolean matched = false;
for (int i = 0; i < whenFilters.size(); i++) {
TupleFilter whenFilter = whenFilters.get(i);
@@ -110,4 +133,21 @@ public class CaseTupleFilter extends TupleFilter {
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
}
+ @Override
+ public TupleFilter acceptOptimizeTransformer(FilterOptimizeTransformer transformer) {
+ List<TupleFilter> newChildren = Lists.newArrayList();
+ for (TupleFilter child : this.getChildren()) {
+ if (child instanceof IOptimizeableTupleFilter) {
+ newChildren.add(((IOptimizeableTupleFilter) child).acceptOptimizeTransformer(transformer));
+ } else {
+ newChildren.add(child);
+ }
+ }
+
+ this.reinit();
+ this.addChildren(newChildren);
+
+ return transformer.visit(this);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d2836e6..e0b18d7 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -32,7 +32,11 @@ import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
/**
* @author xjiang
*/
-public class CompareTupleFilter extends TupleFilter {
+public class CompareTupleFilter extends TupleFilter implements IOptimizeableTupleFilter {
+
+ public enum CompareResultType {
+ AlwaysTrue, AlwaysFalse, Unknown
+ }
// operand 1 is either a column or a function
private TblColRef column;
@@ -225,18 +229,20 @@ public class CompareTupleFilter extends TupleFilter {
&& secondColumn == null;
}
- public boolean alwaysReturnTrue() {
- // 1 = 1
- if (this.operator == FilterOperatorEnum.EQ) {
+ public CompareResultType getCompareResultType() {
+ // cases like 1 = 1, or 'a' <> 'b'
+ if (this.operator == FilterOperatorEnum.EQ || this.operator == FilterOperatorEnum.NEQ) {
if (this.children != null && this.children.size() == 2 && //
this.children.get(0) instanceof ConstantTupleFilter && //
- this.children.get(1) instanceof ConstantTupleFilter && //
- ((ConstantTupleFilter) this.children.get(0)).getValues().equals(((ConstantTupleFilter) this.children.get(1)).getValues())) {
- return true;
+ this.children.get(1) instanceof ConstantTupleFilter) {
+ if (((ConstantTupleFilter) this.children.get(0)).getValues().equals(((ConstantTupleFilter) this.children.get(1)).getValues())) {
+ return this.operator == FilterOperatorEnum.EQ ? CompareResultType.AlwaysTrue : CompareResultType.AlwaysFalse;
+ } else {
+ return this.operator == FilterOperatorEnum.EQ ? CompareResultType.AlwaysFalse : CompareResultType.AlwaysTrue;
+ }
}
}
-
- return false;
+ return CompareResultType.Unknown;
}
@SuppressWarnings({ "unchecked", "rawtypes" })
@@ -262,4 +268,9 @@ public class CompareTupleFilter extends TupleFilter {
}
}
+ @Override
+ public TupleFilter acceptOptimizeTransformer(FilterOptimizeTransformer transformer) {
+ return transformer.visit(this);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
index a3bbad6..9ef9c2c 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -18,45 +18,48 @@
package org.apache.kylin.metadata.filter;
+import java.util.List;
import java.util.ListIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
/**
* optimize the filter if possible, not limited to:
- * <p>
* 1. prune filters like (a = ? OR 1 = 1)
+ *
+ * is a first type transformer defined in ITupleFilterTransformer
*/
public class FilterOptimizeTransformer implements ITupleFilterTransformer {
public static final Logger logger = LoggerFactory.getLogger(FilterOptimizeTransformer.class);
@Override
public TupleFilter transform(TupleFilter tupleFilter) {
- TupleFilter translated = null;
- if (tupleFilter instanceof CompareTupleFilter) {
- //normal case
- translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) tupleFilter);
- } else if (tupleFilter instanceof LogicalTupleFilter) {
- @SuppressWarnings("unchecked")
- ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
- while (childIterator.hasNext()) {
- TupleFilter transformed = transform(childIterator.next());
- if (transformed != null) {
- childIterator.set(transformed);
- } else {
- throw new IllegalStateException("Should not be null");
- }
- }
+ if (tupleFilter == null || !(tupleFilter instanceof IOptimizeableTupleFilter))
+ return tupleFilter;
+ else
+ return ((IOptimizeableTupleFilter) tupleFilter).acceptOptimizeTransformer(this);
+ }
- translated = replaceAlwaysTrueLogicalFilter((LogicalTupleFilter) tupleFilter);
+ public TupleFilter visit(CompareTupleFilter compareTupleFilter) {
+ if (compareTupleFilter != null) {
+ CompareTupleFilter.CompareResultType compareResultType = compareTupleFilter.getCompareResultType();
+ if (compareResultType == CompareTupleFilter.CompareResultType.AlwaysTrue) {
+ logger.debug("Optimize CompareTupleFilter {{}} to ConstantTupleFilter.TRUE", compareTupleFilter);
+ return ConstantTupleFilter.TRUE;
+ } else if (compareResultType == CompareTupleFilter.CompareResultType.AlwaysFalse) {
+ logger.debug("Optimize CompareTupleFilter {{}} to ConstantTupleFilter.FALSE", compareTupleFilter);
+ return ConstantTupleFilter.FALSE;
+ }
}
- return translated == null ? tupleFilter : translated;
+ return compareTupleFilter;
}
- private TupleFilter replaceAlwaysTrueLogicalFilter(LogicalTupleFilter logicalTupleFilter) {
+ public TupleFilter visit(LogicalTupleFilter logicalTupleFilter) {
if (logicalTupleFilter == null) {
return null;
}
@@ -67,23 +70,56 @@ public class FilterOptimizeTransformer implements ITupleFilterTransformer {
while (childIterator.hasNext()) {
TupleFilter next = childIterator.next();
if (ConstantTupleFilter.TRUE == next) {
- logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", logicalTupleFilter);
+ logger.debug("Optimized {{}} to ConstantTupleFilter.TRUE", logicalTupleFilter);
return ConstantTupleFilter.TRUE;
}
}
+ } else if (logicalTupleFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
+ @SuppressWarnings("unchecked")
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) logicalTupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter next = childIterator.next();
+ if (ConstantTupleFilter.FALSE == next) {
+ logger.debug("Optimized {{}} to ConstantTupleFilter.FALSE", logicalTupleFilter);
+ return ConstantTupleFilter.FALSE;
+ }
+ }
}
return logicalTupleFilter;
}
- private TupleFilter replaceAlwaysTrueCompareFilter(CompareTupleFilter compareTupleFilter) {
+ public TupleFilter visit(CaseTupleFilter caseTupleFilter) {
- if (compareTupleFilter != null && compareTupleFilter.alwaysReturnTrue()) {
- logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", compareTupleFilter);
- return ConstantTupleFilter.TRUE;
+ List<TupleFilter> whenFilters = caseTupleFilter.getWhenFilters();
+ List<TupleFilter> thenFilters = caseTupleFilter.getThenFilters();
+ List<TupleFilter> newFilters = Lists.newArrayList();
+ boolean changed = false;
+ for (int i = 0; i < whenFilters.size(); i++) {
+ if (whenFilters.get(i) == ConstantTupleFilter.TRUE) {
+ return thenFilters.get(i);
+ }
+
+ if (whenFilters.get(i) == ConstantTupleFilter.FALSE) {
+ changed = true;
+ continue;
+ }
+
+ newFilters.add(whenFilters.get(i));
+ newFilters.add(thenFilters.get(i));
}
+ newFilters.add(caseTupleFilter.getElseFilter());
- return compareTupleFilter;
+ if (!changed) {
+ return caseTupleFilter;
+ } else {
+ if (newFilters.size() == 1) {
+ return newFilters.get(0);
+ }
+
+ CaseTupleFilter newCaseTupleFilter = new CaseTupleFilter();
+ newCaseTupleFilter.addChildren(newFilters);
+ return newCaseTupleFilter;
+ }
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-metadata/src/main/java/org/apache/kylin/metadata/filter/IOptimizeableTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/IOptimizeableTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/IOptimizeableTupleFilter.java
new file mode 100644
index 0000000..8892035
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/IOptimizeableTupleFilter.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.metadata.filter;
+
+public interface IOptimizeableTupleFilter {
+ TupleFilter acceptOptimizeTransformer(FilterOptimizeTransformer transformer);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
index d3d5076..c8d991e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ITupleFilterTransformer.java
@@ -18,6 +18,11 @@
package org.apache.kylin.metadata.filter;
+/**
+ * there are two types of transformers:
+ * 1. transformation requiring storage-specific details (say segment dict)
+ * 2. others
+ */
public interface ITupleFilterTransformer {
TupleFilter transform(TupleFilter tupleFilter);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 373acdb..7893ed8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -27,19 +27,28 @@ import java.util.List;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-public class LogicalTupleFilter extends TupleFilter {
+import com.google.common.collect.Lists;
+
+public class LogicalTupleFilter extends TupleFilter implements IOptimizeableTupleFilter {
public LogicalTupleFilter(FilterOperatorEnum op) {
super(new ArrayList<TupleFilter>(2), op);
+
boolean opGood = (op == FilterOperatorEnum.AND || op == FilterOperatorEnum.OR || op == FilterOperatorEnum.NOT);
if (opGood == false)
throw new IllegalArgumentException("Unsupported operator " + op);
}
+ //private
private LogicalTupleFilter(List<TupleFilter> filters, FilterOperatorEnum op) {
super(filters, op);
}
+ private void reinitWithChildren(List<TupleFilter> newTupleFilter) {
+ this.children.clear();
+ this.addChildren(newTupleFilter);
+ }
+
@Override
public TupleFilter copy() {
List<TupleFilter> cloneChildren = new LinkedList<TupleFilter>(children);
@@ -145,4 +154,20 @@ public class LogicalTupleFilter extends TupleFilter {
}
+ @Override
+ public TupleFilter acceptOptimizeTransformer(FilterOptimizeTransformer transformer) {
+ List<TupleFilter> newChildren = Lists.newArrayList();
+ for (TupleFilter child : this.getChildren()) {
+ if (child instanceof IOptimizeableTupleFilter) {
+ newChildren.add(((IOptimizeableTupleFilter) child).acceptOptimizeTransformer(transformer));
+ } else {
+ newChildren.add(child);
+ }
+ }
+
+ this.reinitWithChildren(newChildren);
+
+ return transformer.visit(this);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index f9d83f5..5ba8726 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 6c2ff5d..691c506 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -36,6 +36,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.metadata.filter.CaseTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
@@ -85,7 +86,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), request.getMetrics(), request.getFilter(), request.getHavingFilter(), request.getContext());
-
+
if (!scanner.isSegmentSkipped())
scanners.add(scanner);
}
@@ -147,10 +148,10 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
enableStreamAggregateIfBeneficial(cuboid, groupsD, context);
// set query deadline
context.setDeadline(cubeInstance);
-
+
// push down having clause filter if possible
TupleFilter havingFilter = checkHavingCanPushDown(sqlDigest.havingFilter, groupsD, sqlDigest.aggregations, metrics);
-
+
logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation());
return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, havingFilter, context);
@@ -290,8 +291,13 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
LogicalTupleFilter r = new LogicalTupleFilter(filter.getOperator());
r.addChildren(newChildren);
return r;
- } else
+ } else if (filter instanceof CaseTupleFilter) {
+ CaseTupleFilter r = new CaseTupleFilter();
+ r.addChildren(newChildren);
+ return r;
+ } else {
throw new IllegalStateException("Cannot replaceChildren on " + filter);
+ }
}
private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
@@ -424,24 +430,24 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
Segments<CubeSegment> readySegs = cubeInstance.getSegments(SegmentStatusEnum.READY);
if (readySegs.size() != 1)
return null;
-
+
// sharded-by column must on group by
CubeDesc desc = cubeInstance.getDescriptor();
Set<TblColRef> shardBy = desc.getShardByColumns();
if (groupsD == null || shardBy.isEmpty() || !groupsD.containsAll(shardBy))
return null;
-
+
// OK, push down
logger.info("Push down having filter " + havingFilter);
-
+
// convert columns in the filter
Set<TblColRef> aggrOutCols = new HashSet<>();
TupleFilter.collectColumns(havingFilter, aggrOutCols);
-
+
for (TblColRef aggrOutCol : aggrOutCols) {
int aggrIdxOnSql = aggrOutCol.getColumnDesc().getZeroBasedIndex(); // aggr index marked in OLAPAggregateRel
FunctionDesc aggrFunc = aggregations.get(aggrIdxOnSql);
-
+
// calculate the index of this aggr among all the metrics that is sending to storage
int aggrIdxAmongMetrics = 0;
for (MeasureDesc m : cubeDesc.getMeasures()) {
@@ -452,7 +458,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
aggrOutCol.getColumnDesc().setId("" + (aggrIdxAmongMetrics + 1));
}
-
+
return havingFilter;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 0833a92..8f86ae0 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -379,7 +379,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
TupleFilter filter = this.condition.accept(visitor);
// optimize the filter, the optimization has to be segment-irrelevant
- new FilterOptimizeTransformer().transform(filter);
+ filter = new FilterOptimizeTransformer().transform(filter);
Set<TblColRef> filterColumns = Sets.newHashSet();
TupleFilter.collectColumns(filter, filterColumns);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 1b5970c..a27cf76 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -107,6 +107,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
return super.estimateRowCount(mq) * 0.1;
}
+ //when OLAPJoinPushThroughJoinRule is applied, a "MerelyPermutation" project rel will be created
private boolean isParentMerelyPermutation(OLAPImplementor implementor) {
if (implementor.getParentNode() instanceof OLAPProjectRel) {
return ((OLAPProjectRel) implementor.getParentNode()).isMerelyPermutation();
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d794046/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
index b0336db..059e167 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/UserController.java
@@ -61,12 +61,12 @@ public class UserController extends BasicController {
}
if (authentication.getPrincipal() instanceof UserDetails) {
- logger.debug("authentication.getPrincipal() is " + authentication.getPrincipal());
+ //logger.debug("authentication.getPrincipal() is " + authentication.getPrincipal());
return (UserDetails) authentication.getPrincipal();
}
if (authentication.getDetails() instanceof UserDetails) {
- logger.debug("authentication.getDetails() is " + authentication.getDetails());
+ //logger.debug("authentication.getDetails() is " + authentication.getDetails());
return (UserDetails) authentication.getDetails();
}
[2/2] kylin git commit: KYLIN-2636 optimize case when in group by
Posted by li...@apache.org.
KYLIN-2636 optimize case when in group by
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/94cd373f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/94cd373f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/94cd373f
Branch: refs/heads/2.0.x
Commit: 94cd373fba89784263c25bc71ecdf612a7c333cf
Parents: 5d79404
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon May 22 16:45:24 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon May 22 19:42:24 2017 +0800
----------------------------------------------------------------------
.../resources/query/sql_subquery/query32.sql | 14 +++++
.../resources/query/sql_subquery/query33.sql | 14 +++++
.../resources/query/sql_subquery/query34.sql | 14 +++++
.../kylin/query/relnode/OLAPProjectRel.java | 61 +++++++++++++++++++-
4 files changed, 102 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/94cd373f/kylin-it/src/test/resources/query/sql_subquery/query32.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query32.sql b/kylin-it/src/test/resources/query/sql_subquery/query32.sql
new file mode 100644
index 0000000..394ea15
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query32.sql
@@ -0,0 +1,14 @@
+
+ select sum(price) as sum_price, (case when '1'='1' then test_cal_dt.week_beg_dt when '1'='2' then test_kylin_fact.lstg_site_id else test_kylin_fact.leaf_categ_id end) as xxx
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where case when '1'='1' then test_kylin_fact.cal_dt < date'2012-04-01' when '1'='2' then test_cal_dt.week_beg_dt > date'2012-04-01' else test_kylin_fact.lstg_site_id is not null end and lstg_format_name='FP-GTC'
+
+ group by case when '1'='1' then test_cal_dt.week_beg_dt when '1'='2' then test_kylin_fact.lstg_site_id else test_kylin_fact.leaf_categ_id end
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/94cd373f/kylin-it/src/test/resources/query/sql_subquery/query33.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query33.sql b/kylin-it/src/test/resources/query/sql_subquery/query33.sql
new file mode 100644
index 0000000..08ecc8e
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query33.sql
@@ -0,0 +1,14 @@
+
+ select sum(price) as sum_price, lstg_format_name,(case when '2'='1' then test_kylin_fact.lstg_site_id when '2'='2' then test_cal_dt.week_beg_dt else test_kylin_fact.leaf_categ_id end) as xxx
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where case when '2'='1' then test_cal_dt.week_beg_dt > date'2012-04-01' when '2'='2' then test_kylin_fact.cal_dt < date'2012-04-01' else test_kylin_fact.lstg_site_id is not null end
+
+ group by case when '2'='1' then test_kylin_fact.lstg_site_id when '2'='2' then test_cal_dt.week_beg_dt else test_kylin_fact.leaf_categ_id end,lstg_format_name
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/94cd373f/kylin-it/src/test/resources/query/sql_subquery/query34.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query34.sql b/kylin-it/src/test/resources/query/sql_subquery/query34.sql
new file mode 100644
index 0000000..0dda9aa
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query34.sql
@@ -0,0 +1,14 @@
+
+ select sum(price) as sum_price, (case when '3'='1' then test_cal_dt.week_beg_dt else test_kylin_fact.cal_dt end) as xxx
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where case when '3'='1' then test_kylin_fact.cal_dt < date'2012-04-01' else test_cal_dt.week_beg_dt > date'2012-04-01' end and lstg_format_name='FP-GTC'
+
+ group by case when '3'='1' then test_cal_dt.week_beg_dt else test_kylin_fact.cal_dt end
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/94cd373f/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index 399a337..b749d85 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -18,6 +18,8 @@
package org.apache.kylin.query.relnode;
+import static org.apache.kylin.metadata.filter.CompareTupleFilter.CompareResultType;
+
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
@@ -46,13 +48,16 @@ import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexProgram;
+import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlCaseOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
/**
*/
@@ -208,12 +213,66 @@ public class OLAPProjectRel extends Project implements OLAPRel {
}
}
- for (RexNode operand : call.getOperands()) {
+ List<RexNode> children = limitTranslateScope(call.getOperands(), operator);
+
+ for (RexNode operand : children) {
translateRexNode(operand, inputColumnRowType, fieldName, sourceCollector);
}
return TblColRef.newInnerColumn(fieldName, InnerDataTypeEnum.LITERAL, call.toString());
}
+ //in most cases it will return children itself
+ private List<RexNode> limitTranslateScope(List<RexNode> children, SqlOperator operator) {
+
+ //group by case when 1 = 1 then x 1 = 2 then y else z
+ if (operator instanceof SqlCaseOperator) {
+ int unknownWhenCalls = 0;
+ for (int i = 0; i < children.size() - 1; i += 2) {
+ if (children.get(i) instanceof RexCall) {
+ RexCall whenCall = (RexCall) children.get(i);
+ CompareResultType compareResultType = getCompareResultType(whenCall);
+ if (compareResultType == CompareResultType.AlwaysTrue) {
+ return Lists.newArrayList(children.get(i), children.get(i + 1));
+ } else if (compareResultType == CompareResultType.Unknown) {
+ unknownWhenCalls++;
+ }
+ }
+ }
+
+ if (unknownWhenCalls == 0) {
+ return Lists.newArrayList(children.get(children.size() - 1));
+ }
+ }
+
+ return children;
+ }
+
+ private CompareResultType getCompareResultType(RexCall whenCall) {
+ List<RexNode> operands = whenCall.getOperands();
+ if (SqlKind.EQUALS == whenCall.getKind() && operands != null && operands.size() == 2) {
+ if (operands.get(0).equals(operands.get(1))) {
+ return CompareResultType.AlwaysTrue;
+ }
+
+ if (isConstant(operands.get(0)) && isConstant(operands.get(0))) {
+ return CompareResultType.AlwaysFalse;
+ }
+ }
+ return CompareResultType.Unknown;
+ }
+
+ private boolean isConstant(RexNode rexNode) {
+ if (rexNode instanceof RexLiteral) {
+ return true;
+ }
+
+ if (rexNode instanceof RexCall && SqlKind.CAST.equals(rexNode.getKind()) && ((RexCall) rexNode).getOperands().get(0) instanceof RexLiteral) {
+ return true;
+ }
+
+ return false;
+ }
+
@Override
public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
if (getInput() instanceof OLAPFilterRel) {