You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ka...@apache.org on 2017/04/13 11:22:02 UTC
[36/50] [abbrv] kylin git commit: KYLIN-2539 Useless filter dimension
will impact cuboid selection.
KYLIN-2539 Useless filter dimension will impact cuboid selection.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/177079d1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/177079d1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/177079d1
Branch: refs/heads/KYLIN-2506
Commit: 177079d10c1dfd4decadd743eafdfc36a712e470
Parents: c17b3eb
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Apr 10 21:21:41 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Apr 10 21:22:09 2017 +0800
----------------------------------------------------------------------
.../metadata/filter/CompareTupleFilter.java | 14 +++
.../filter/FilterOptimizeTransformer.java | 89 ++++++++++++++++++++
.../gtrecord/GTCubeStorageQueryBase.java | 6 +-
3 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index d567c89..d783e52 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -218,6 +218,20 @@ public class CompareTupleFilter extends TupleFilter {
&& !conditionValues.isEmpty() && secondColumn == null;
}
+ public boolean alwaysReturnTrue() {
+ // 1 = 1
+ if (this.operator == FilterOperatorEnum.EQ) {
+ if (this.children != null && this.children.size() == 2 && //
+ this.children.get(0) instanceof ConstantTupleFilter && //
+ this.children.get(1) instanceof ConstantTupleFilter && //
+ ((ConstantTupleFilter) this.children.get(0)).getValues().equals(((ConstantTupleFilter) this.children.get(1)).getValues())) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public void serialize(IFilterCodeSystem cs, ByteBuffer buffer) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
new file mode 100755
index 0000000..a3bbad6
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/FilterOptimizeTransformer.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * optimize the filter if possible, not limited to:
+ * <p>
+ * 1. prune filters like (a = ? OR 1 = 1)
+ */
+public class FilterOptimizeTransformer implements ITupleFilterTransformer {
+ public static final Logger logger = LoggerFactory.getLogger(FilterOptimizeTransformer.class);
+
+ @Override
+ public TupleFilter transform(TupleFilter tupleFilter) {
+ TupleFilter translated = null;
+ if (tupleFilter instanceof CompareTupleFilter) {
+ //normal case
+ translated = replaceAlwaysTrueCompareFilter((CompareTupleFilter) tupleFilter);
+ } else if (tupleFilter instanceof LogicalTupleFilter) {
+ @SuppressWarnings("unchecked")
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter transformed = transform(childIterator.next());
+ if (transformed != null) {
+ childIterator.set(transformed);
+ } else {
+ throw new IllegalStateException("Should not be null");
+ }
+ }
+
+ translated = replaceAlwaysTrueLogicalFilter((LogicalTupleFilter) tupleFilter);
+
+ }
+ return translated == null ? tupleFilter : translated;
+
+ }
+
+ private TupleFilter replaceAlwaysTrueLogicalFilter(LogicalTupleFilter logicalTupleFilter) {
+ if (logicalTupleFilter == null) {
+ return null;
+ }
+
+ if (logicalTupleFilter.getOperator() == TupleFilter.FilterOperatorEnum.OR) {
+ @SuppressWarnings("unchecked")
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) logicalTupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter next = childIterator.next();
+ if (ConstantTupleFilter.TRUE == next) {
+ logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", logicalTupleFilter);
+ return ConstantTupleFilter.TRUE;
+ }
+ }
+ }
+
+ return logicalTupleFilter;
+ }
+
+ private TupleFilter replaceAlwaysTrueCompareFilter(CompareTupleFilter compareTupleFilter) {
+
+ if (compareTupleFilter != null && compareTupleFilter.alwaysReturnTrue()) {
+ logger.debug("Translated {{}} to ConstantTupleFilter.TRUE", compareTupleFilter);
+ return ConstantTupleFilter.TRUE;
+ }
+
+ return compareTupleFilter;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/177079d1/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 065fd6e..abac33a 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -38,6 +38,7 @@ import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
@@ -148,13 +149,16 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// set query deadline
context.setDeadline(cubeInstance);
+ // optimize the filter, the optimization has to be segment-irrelevant
+ new FilterOptimizeTransformer().transform(filterD);
+
logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation());
return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, context);
}
protected abstract String getGTStorage();
-
+
protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo tupleInfo) {
return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, gtColIdx, tupleInfo);
}