You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/07/27 03:30:39 UTC
kylin git commit: KYLIN-1922 Improve the logic to decide whether to
pre aggregate on Region server
Repository: kylin
Updated Branches:
refs/heads/beforepomupdate a055b8891 -> 70dc19047
KYLIN-1922 Improve the logic to decide whether to pre aggregate on Region server
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70dc1904
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70dc1904
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70dc1904
Branch: refs/heads/beforepomupdate
Commit: 70dc190477d4b1233e36fba59c58970a5f8fc1a4
Parents: a055b88
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jul 27 11:09:26 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jul 27 11:09:26 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTScanRequest.java | 12 +++++++++++-
.../java/org/apache/kylin/storage/StorageContext.java | 9 +++++++++
.../kylin/storage/gtrecord/CubeSegmentScanner.java | 2 +-
.../kylin/storage/gtrecord/GTCubeStorageQueryBase.java | 12 ++++++++++--
4 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/70dc1904/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index d4e6298..d1bd1b3 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -33,12 +33,16 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.SerializeToByteBuffer;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
public class GTScanRequest {
+ private static final Logger logger = LoggerFactory.getLogger(GTScanRequest.class);
+
private GTInfo info;
private List<GTScanRange> ranges;
private ImmutableBitSet columns;
@@ -165,11 +169,17 @@ public class GTScanRequest {
return new EmptyGTScanner(scanned);
}
- if (this.allowPreAggregation && this.hasAggregation()) {
+ if (!this.allowPreAggregation) {
+ logger.info("pre aggregation is not beneficial, skip it");
+ } else if (this.hasAggregation()) {
+ logger.info("pre aggregating results before returning");
result = new GTAggregateScanner(result, this);
+ } else {
+ logger.info("has no aggregation, skip it");
}
return result;
}
+
}
//touch every byte of the cell so that the cost of scanning will be truly reflected
http://git-wip-us.apache.org/repos/asf/kylin/blob/70dc1904/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index 90f950c..aad37ce 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -39,6 +39,7 @@ public class StorageContext {
private boolean acceptPartialResult;
private boolean exactAggregation;
+ private boolean groupingByAllCuboidDimensions;
private boolean enableLimit;
private boolean enableCoprocessor;
@@ -143,6 +144,14 @@ public class StorageContext {
this.partialResultReturned = partialResultReturned;
}
+ public boolean isGroupingByAllCuboidDimensions() {
+ return groupingByAllCuboidDimensions;
+ }
+
+ public void setGroupingByAllCuboidDimensions(boolean groupingByAllCuboidDimensions) {
+ this.groupingByAllCuboidDimensions = groupingByAllCuboidDimensions;
+ }
+
public void setExactAggregation(boolean isExactAggregation) {
this.exactAggregation = isExactAggregation;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/70dc1904/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 15e3272..f539179 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -69,7 +69,7 @@ public class CubeSegmentScanner implements IGTScanner {
}
scanRequest = scanRangePlanner.planScanRequest();
if (scanRequest != null) {
- scanRequest.setAllowPreAggregation(!context.isExactAggregation());
+ scanRequest.setAllowPreAggregation(!context.isGroupingByAllCuboidDimensions());
scanRequest.setAggrCacheGB(cubeSeg.getCubeInstance().getConfig().getQueryCoprocessorMemGB());
if (context.isLimitEnabled())
scanRequest.setRowLimit(context.getLimit());
http://git-wip-us.apache.org/repos/asf/kylin/blob/70dc1904/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 65aa90a..d03cfc6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -99,8 +99,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// isExactAggregation? meaning: tuples returned from storage requires no further aggregation in query engine
Set<TblColRef> singleValuesD = findSingleValueColumns(filter);
- boolean isExactAggregation = isExactAggregation(cuboid, groups, otherDimsD, singleValuesD, derivedPostAggregation);
- context.setExactAggregation(isExactAggregation);
+ context.setExactAggregation(isExactAggregation(cuboid, groups, otherDimsD, singleValuesD, derivedPostAggregation));
+ context.setGroupingByAllCuboidDimensions(isGroupingByAllCuboidDimensions(cuboid, groupsD));
// replace derived columns in filter with host columns; columns on loosened condition must be added to group by
TupleFilter filterD = translateDerived(filter, groupsD);
@@ -221,6 +221,14 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return resultD;
}
+ private boolean isGroupingByAllCuboidDimensions(Cuboid cuboid, Collection<TblColRef> groupD) {
+ if (cuboid.getColumns().size() != groupD.size()) {
+ return false;
+ }
+
+ return groupD.containsAll(cuboid.getColumns());
+ }
+
private boolean isExactAggregation(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation) {
boolean exact = true;