You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ka...@apache.org on 2017/04/13 11:21:37 UTC
[11/50] [abbrv] kylin git commit: minor,
code refactor in GTCubeStorageQueryBase
minor, code refactor in GTCubeStorageQueryBase
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4cce762
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4cce762
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4cce762
Branch: refs/heads/KYLIN-2506
Commit: d4cce762addda1c819939e767220864dad551503
Parents: f6ee76c
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 30 17:25:22 2017 +0800
Committer: hongbin ma <ma...@kyligence.io>
Committed: Fri Mar 31 18:34:37 2017 +0800
----------------------------------------------------------------------
.../gtrecord/GTCubeStorageQueryBase.java | 68 +++++++-------
.../gtrecord/GTCubeStorageQueryRequest.java | 94 ++++++++++++++++++++
2 files changed, 130 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4cce762/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index ecf1ad3..1b4964f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -72,6 +72,30 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);
+
+ List<CubeSegmentScanner> scanners = Lists.newArrayList();
+ for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
+ CubeSegmentScanner scanner;
+
+ if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
+ logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
+ continue;
+ }
+
+ scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), request.getMetrics(), request.getFilter(), request.getContext());
+ scanners.add(scanner);
+ }
+
+ if (scanners.isEmpty())
+ return ITupleIterator.EMPTY_TUPLE_ITERATOR;
+
+ return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(), request.getMetrics(), returnTupleInfo, request.getContext());
+ }
+
+ protected abstract String getGTStorage();
+
+ protected GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
context.setStorageQuery(this);
//deal with participant columns in subquery join
@@ -106,7 +130,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
dimensionsD.addAll(groupsD);
dimensionsD.addAll(otherDimsD);
Cuboid cuboid = Cuboid.identifyCuboid(cubeDesc, dimensionsD, metrics);
- logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, otherDimsD={}", cubeInstance.getName(), cuboid.getId(), groupsD, otherDimsD);
context.setCuboid(cuboid);
// set whether to aggr at storage
@@ -115,35 +138,21 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// replace derived columns in filter with host columns; columns on loosened condition must be added to group by
Set<TblColRef> loosenedColumnD = Sets.newHashSet();
+ Set<TblColRef> filterColumnD = Sets.newHashSet();
TupleFilter filterD = translateDerived(filter, loosenedColumnD);
groupsD.addAll(loosenedColumnD);
+ TupleFilter.collectColumns(filter, filterColumnD);
// set limit push down
enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context);
// set query deadline
context.setDeadline(cubeInstance);
- List<CubeSegmentScanner> scanners = Lists.newArrayList();
- for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
- CubeSegmentScanner scanner;
-
- if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
- logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
- continue;
- }
-
- scanner = new CubeSegmentScanner(cubeSeg, cuboid, dimensionsD, groupsD, metrics, filterD, context);
- scanners.add(scanner);
- }
+ logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation());
- if (scanners.isEmpty())
- return ITupleIterator.EMPTY_TUPLE_ITERATOR;
-
- return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
+ return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, context);
}
- protected abstract String getGTStorage();
-
protected ITupleConverter newCubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo tupleInfo) {
return new CubeTupleConverter(cubeSeg, cuboid, selectedDimensions, selectedMetrics, tupleInfo);
}
@@ -235,19 +244,14 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
public boolean isNeedStorageAggregation(Cuboid cuboid, Collection<TblColRef> groupD, Collection<TblColRef> singleValueD) {
-
- logger.info("GroupD :" + groupD);
- logger.info("SingleValueD :" + singleValueD);
- logger.info("Cuboid columns :" + cuboid.getColumns());
-
HashSet<TblColRef> temp = Sets.newHashSet();
temp.addAll(groupD);
temp.addAll(singleValueD);
if (cuboid.getColumns().size() == temp.size()) {
- logger.info("Does not need storage aggregation");
+ logger.debug("Does not need storage aggregation");
return false;
} else {
- logger.info("Need storage aggregation");
+ logger.debug("Need storage aggregation");
return true;
}
}
@@ -326,30 +330,30 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
if (!TupleFilter.isEvaluableRecursively(filter)) {
possible = false;
- logger.info("Storage limit push down is impossible because the filter isn't evaluable");
+ logger.debug("Storage limit push down is impossible because the filter isn't evaluable");
}
if (!loosenedColumnD.isEmpty()) { // KYLIN-2173
possible = false;
- logger.info("Storage limit push down is impossible because filter is loosened: " + loosenedColumnD);
+ logger.debug("Storage limit push down is impossible because filter is loosened: " + loosenedColumnD);
}
if (context.hasSort()) {
possible = false;
- logger.info("Storage limit push down is impossible because the query has order by");
+ logger.debug("Storage limit push down is impossible because the query has order by");
}
// derived aggregation is bad, unless expanded columns are already in group by
if (!groups.containsAll(derivedPostAggregation)) {
possible = false;
- logger.info("Storage limit push down is impossible because derived column require post aggregation: " + derivedPostAggregation);
+ logger.debug("Storage limit push down is impossible because derived column require post aggregation: " + derivedPostAggregation);
}
//if groupsD is clustered at "head" of the rowkey, then limit push down is possible
int size = groupsD.size();
if (!groupsD.containsAll(cuboid.getColumns().subList(0, size))) {
possible = false;
- logger.info("Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD //
+ logger.debug("Storage limit push down is impossible because groupD is not clustered at head, groupsD: " + groupsD //
+ " with cuboid columns: " + cuboid.getColumns());
}
@@ -357,7 +361,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
for (FunctionDesc functionDesc : functionDescs) {
if (functionDesc.isDimensionAsMetric()) {
possible = false;
- logger.info("Storage limit push down is impossible because {} isDimensionAsMetric ", functionDesc);
+ logger.debug("Storage limit push down is impossible because {} isDimensionAsMetric ", functionDesc);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4cce762/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
new file mode 100644
index 0000000..c4d81d2
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import java.io.Serializable;
+import java.util.Set;
+
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.StorageContext;
+
+public class GTCubeStorageQueryRequest implements Serializable {
+ private Cuboid cuboid;
+ private Set<TblColRef> dimensions;
+ private Set<TblColRef> groups;
+ private Set<FunctionDesc> metrics;
+ private TupleFilter filter;
+ private StorageContext context;
+
+ public GTCubeStorageQueryRequest(Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics, TupleFilter filter, StorageContext context) {
+ this.cuboid = cuboid;
+ this.dimensions = dimensions;
+ this.groups = groups;
+ this.metrics = metrics;
+ this.filter = filter;
+ this.context = context;
+ }
+
+ public Cuboid getCuboid() {
+ return cuboid;
+ }
+
+ public void setCuboid(Cuboid cuboid) {
+ this.cuboid = cuboid;
+ }
+
+ public Set<TblColRef> getDimensions() {
+ return dimensions;
+ }
+
+ public void setDimensions(Set<TblColRef> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ public Set<TblColRef> getGroups() {
+ return groups;
+ }
+
+ public void setGroups(Set<TblColRef> groups) {
+ this.groups = groups;
+ }
+
+ public Set<FunctionDesc> getMetrics() {
+ return metrics;
+ }
+
+ public void setMetrics(Set<FunctionDesc> metrics) {
+ this.metrics = metrics;
+ }
+
+ public TupleFilter getFilter() {
+ return filter;
+ }
+
+ public void setFilter(TupleFilter filter) {
+ this.filter = filter;
+ }
+
+ public StorageContext getContext() {
+ return context;
+ }
+
+ public void setContext(StorageContext context) {
+ this.context = context;
+ }
+}