You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/20 11:26:37 UTC
[41/50] [abbrv] kylin git commit: KYLIN-2290 minor improvements on
limit
KYLIN-2290 minor improvements on limit
KYLIN-2290 fix compile
KYLIN-2290 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f9bd4a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f9bd4a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f9bd4a9
Branch: refs/heads/master-hbase1.x
Commit: 6f9bd4a9998cff24c97bce0e6d86282f48fef388
Parents: 699a88e
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Dec 16 16:28:18 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Dec 19 14:16:00 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 1 +
.../kylin/cube/CubeCapabilityChecker.java | 5 ++
.../kylin/gridtable/GTScanRequestBuilder.java | 2 +-
.../kylin/metadata/realization/SQLDigest.java | 4 +-
.../apache/kylin/storage/StorageContext.java | 74 +++++++-------------
.../storage/gtrecord/CubeScanRangePlanner.java | 9 +--
.../gtrecord/GTCubeStorageQueryBase.java | 1 -
.../gtrecord/SequentialCubeTupleIterator.java | 6 +-
.../apache/kylin/query/ITKylinQueryTest.java | 1 -
.../org/apache/kylin/query/KylinTestBase.java | 7 +-
.../kylin/storage/hbase/ITStorageTest.java | 2 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 4 ++
.../apache/kylin/query/relnode/OLAPContext.java | 6 +-
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../kylin/query/relnode/OLAPLimitRel.java | 6 +-
.../coprocessor/endpoint/CubeVisitService.java | 7 +-
16 files changed, 60 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 610c2af..5153562 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -778,6 +778,7 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
}
+ @Deprecated//Limit is good even it's large. This config is meaning less since we already have scan threshold
public int getStoragePushDownLimitMax() {
return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 38faed9..c45144b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -120,6 +120,11 @@ public class CubeCapabilityChecker {
return result;
}
+ if (digest.limitPrecedesAggr) {
+ logger.info("Exclude cube " + cube.getName() + " because there's limit preceding aggregation");
+ return result;
+ }
+
if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) {
result.influences.add(new CapabilityInfluence() {
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index f542de1..bcec1f4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -36,7 +36,7 @@ public class GTScanRequestBuilder {
private boolean allowStorageAggregation = true;
private double aggCacheMemThreshold = 0;
private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage should terminate itself when $storageScanRowNumThreshold cuboid rows are scanned, and throw exception.
- private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit working when $toragePushDownLimit aggregated rows are produced.
+ private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit scanning safely when $toragePushDownLimit aggregated rows are produced.
private long startTime = -1;
private long timeout = -1;
private String storageBehavior = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 83fc05c..36f303b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -59,10 +59,11 @@ public class SQLDigest {
public List<TblColRef> sortColumns;
public List<OrderEnum> sortOrders;
public boolean isRawQuery;
+ public boolean limitPrecedesAggr;
public SQLDigest(String factTable, TupleFilter filter, List<JoinDesc> joinDescs, Set<TblColRef> allColumns, //
List<TblColRef> groupbyColumns, Set<TblColRef> subqueryJoinParticipants, Set<TblColRef> filterColumns, Set<TblColRef> metricColumns, //
- List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders) {
+ List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders, boolean limitPrecedesAggr) {
this.factTable = factTable;
this.filter = filter;
this.joinDescs = joinDescs;
@@ -76,6 +77,7 @@ public class SQLDigest {
this.sortColumns = sortColumns;
this.sortOrders = sortOrders;
this.isRawQuery = isRawQuery();
+ this.limitPrecedesAggr = limitPrecedesAggr;
}
private boolean isRawQuery() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index bc43a87..9ef59fd 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -34,43 +34,29 @@ import com.google.common.collect.Range;
public class StorageContext {
private static final Logger logger = LoggerFactory.getLogger(StorageContext.class);
- public static final int DEFAULT_THRESHOLD = 1000000;
-
private String connUrl;
private int threshold;
- private int limit;
- private int offset;
- private int finalPushDownLimit;
- private boolean hasSort;
- private boolean acceptPartialResult;
-
- private boolean exactAggregation;
- private boolean needStorageAggregation;
- private boolean enableLimit;
- private boolean enableCoprocessor;
-
- private AtomicLong totalScanCount;
+ private int limit = Integer.MAX_VALUE;
+ private int offset = 0;
+ private int finalPushDownLimit = Integer.MAX_VALUE;
+ private boolean hasSort = false;
+ private boolean acceptPartialResult = false;
+
+ private boolean exactAggregation = false;
+ private boolean needStorageAggregation = false;
+ private boolean limitEnabled = false;
+ private boolean enableCoprocessor = false;
+
+ private AtomicLong totalScanCount = new AtomicLong();
private Cuboid cuboid;
- private boolean partialResultReturned;
-
- private Range<Long> reusedPeriod;
+ private boolean partialResultReturned = false;
public StorageContext() {
- this.threshold = DEFAULT_THRESHOLD;
- this.limit = DEFAULT_THRESHOLD;
- this.totalScanCount = new AtomicLong();
- this.cuboid = null;
- this.hasSort = false;
-
- this.exactAggregation = false;
- this.enableLimit = false;
- this.enableCoprocessor = false;
-
- this.acceptPartialResult = false;
- this.partialResultReturned = false;
- this.finalPushDownLimit = Integer.MAX_VALUE;
+ this.threshold = KylinConfig.getInstanceFromEnv().getScanThreshold();
}
+ private Range<Long> reusedPeriod;
+
public String getConnUrl() {
return connUrl;
}
@@ -92,11 +78,10 @@ public class StorageContext {
}
public void setLimit(int l) {
- if (l > limit) {
- //cases like : select price from (select * from kylin_sales limit 10) limit 5000
- logger.info("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit);
+ if (limit != Integer.MAX_VALUE) {
+ logger.warn("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit);
} else {
- this.limit = l;
+ limit = l;
}
}
@@ -109,15 +94,11 @@ public class StorageContext {
}
public void enableLimit() {
- this.enableLimit = true;
+ this.limitEnabled = true;
}
public boolean isLimitEnabled() {
- return this.enableLimit;
- }
-
- private int getStoragePushDownLimit() {
- return this.isLimitEnabled() ? this.getOffset() + this.getLimit() : Integer.MAX_VALUE;
+ return this.limitEnabled;
}
public int getFinalPushDownLimit() {
@@ -126,19 +107,16 @@ public class StorageContext {
public void setFinalPushDownLimit(IRealization realization) {
- //decide the final limit push down
- int tempPushDownLimit = this.getStoragePushDownLimit();
- if (tempPushDownLimit == Integer.MAX_VALUE) {
+ if (this.getLimit() == Integer.MAX_VALUE) {
return;
}
- int pushDownLimitMax = KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax();
+ int tempPushDownLimit = this.getOffset() + this.getLimit();
+
if (!realization.supportsLimitPushDown()) {
- logger.info("Not enabling limit push down because cube storage type not supported");
- } else if (tempPushDownLimit > pushDownLimitMax) {
- logger.info("Not enabling limit push down because the limit(including offset) {} is larger than kylin.query.max-limit-pushdown {}", //
- tempPushDownLimit, pushDownLimitMax);
+ logger.warn("Not enabling limit push down because cube storage type not supported");
} else {
+ this.limitEnabled = true;
this.finalPushDownLimit = tempPushDownLimit;
logger.info("Enable limit: " + tempPushDownLimit);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
index 8d5a3d4..b05a629 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -148,15 +148,10 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase {
GTScanRequest scanRequest;
List<GTScanRange> scanRanges = this.planScanRanges();
if (scanRanges != null && scanRanges.size() != 0) {
- GTScanRequestBuilder builder = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).//
+ scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).//
setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).//
setAllowStorageAggregation(context.isNeedStorageAggregation()).setAggCacheMemThreshold(cubeSegment.getCubeInstance().getConfig().getQueryCoprocessorMemGB()).//
- setStorageScanRowNumThreshold(context.getThreshold());
-
- if (context.getFinalPushDownLimit() != Integer.MAX_VALUE)
- builder.setStoragePushDownLimit(context.getFinalPushDownLimit());
-
- scanRequest = builder.createGTScanRequest();
+ setStoragePushDownLimit(context.getFinalPushDownLimit()).setStorageScanRowNumThreshold(context.getThreshold()).createGTScanRequest();
} else {
scanRequest = null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 31573d0..7b0cbb9 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -383,7 +383,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
if (possible) {
- context.enableLimit();
context.setFinalPushDownLimit(cubeInstance);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index bef0e88..3a64de7 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -46,7 +46,6 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
protected List<CubeSegmentScanner> scanners;
protected List<SegmentCubeTupleIterator> segmentCubeTupleIterators;
protected Iterator<ITuple> tupleIterator;
- protected final int storagePushDownLimit;
protected StorageContext context;
private int scanCount;
@@ -62,8 +61,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context));
}
- this.storagePushDownLimit = context.getFinalPushDownLimit();
- if (storagePushDownLimit == Integer.MAX_VALUE) {
+ if (!context.isLimitEnabled()) {
//normal case
tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
} else {
@@ -75,7 +73,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
return input;
}
});
- tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, storagePushDownLimit, segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator();
+ tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 7f04bbb..c2aeabd 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -331,7 +331,6 @@ public class ITKylinQueryTest extends KylinTestBase {
List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql");
for (File sqlFile : sqlFiles) {
runSQL(sqlFile, false, false);
- assertTrue(checkLimitEnabled());
assertTrue(checkFinalPushDownLimit());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index ddb996c..47b9e37 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -653,14 +653,9 @@ public class KylinTestBase {
}
- protected boolean checkLimitEnabled() {
- OLAPContext context = getFirstOLAPContext();
- return (context.storageContext.isLimitEnabled());
- }
-
protected boolean checkFinalPushDownLimit() {
OLAPContext context = getFirstOLAPContext();
- return (context.storageContext.getFinalPushDownLimit() != Integer.MAX_VALUE);
+ return context.storageContext.isLimitEnabled();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
index db9d133..aea8bef 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
@@ -148,7 +148,7 @@ public class ITStorageTest extends HBaseMetadataTestCase {
int count = 0;
ITupleIterator iterator = null;
try {
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>());
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false);
iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
while (iterator.hasNext()) {
ITuple tuple = iterator.next();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index f5fa74d..c457545 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -156,6 +156,10 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
this.context.groupByColumns.addAll(this.groups);
this.context.aggregations.addAll(this.aggregations);
this.context.afterAggregate = true;
+
+ if (this.context.afterLimit) {
+ this.context.limitPrecedesAggr = true;
+ }
} else {
for (AggregateCall aggCall : aggCalls) {
// check if supported by kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 8278fb0..dde98a6 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -112,7 +112,9 @@ public class OLAPContext {
public Set<OLAPTableScan> allTableScans = new HashSet<>();
public TupleInfo returnTupleInfo = null;
public boolean afterAggregate = false;
- public boolean afterSkippedFilter = false;
+ public boolean afterHavingClauseFilter = false;
+ public boolean afterLimit = false;
+ public boolean limitPrecedesAggr = false;
public boolean afterJoin = false;
public boolean hasJoin = false;
@@ -148,7 +150,7 @@ public class OLAPContext {
public SQLDigest getSQLDigest() {
if (sqlDigest == null)
- sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders);
+ sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders, limitPrecedesAggr);
return sqlDigest;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 8023df4..151131e 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -306,7 +306,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
if (!context.afterAggregate) {
translateFilter(context);
} else {
- context.afterSkippedFilter = true;//having clause is skipped
+ context.afterHavingClauseFilter = true;//having clause is skipped
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index f0af863..8179807 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -77,7 +77,9 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
this.columnRowType = buildColumnRowType();
this.context = implementor.getContext();
- if (!context.afterSkippedFilter) {
+ // ignore limit after having clause
+ // ignore limit after another limit, e.g. select A, count(*) from (select A,B from fact group by A,B limit 100) limit 10
+ if (!context.afterHavingClauseFilter && !context.afterLimit) {
Number limitValue = (Number) (((RexLiteral) localFetch).getValue());
int limit = limitValue.intValue();
this.context.storageContext.setLimit(limit);
@@ -87,6 +89,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
int offset = offsetValue.intValue();
this.context.storageContext.setOffset(offset);
}
+
+ context.afterLimit = true;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index da9c932..38efecc 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -254,14 +254,15 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
@Override
public boolean hasNext() {
+ counter++;
+
if (counter > scanReq.getStorageScanRowNumThreshold()) {
- throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter);
+ throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter + ", consider increasing kylin.query.memory-budget-bytes and kylin.query.scan-threshold");
}
if (counter % (10 * GTScanRequest.terminateCheckInterval) == 1) {
- logger.info("Scanned " + counter + " rows from HBase.");
+ logger.info("scanning " + counter + "th row from HBase.");
}
- counter++;
return allCellLists.hasNext();
}