You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/17 03:30:25 UTC
[08/20] kylin git commit: KYLIN-2238 Add query server scan threshold
KYLIN-2238 Add query server scan threshold
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b00ec26
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b00ec26
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b00ec26
Branch: refs/heads/master-hbase1.x
Commit: 4b00ec26de1e1cc9bf9b2e55a1b162d123114df8
Parents: 20c3294
Author: kangkaisen <ka...@live.com>
Authored: Tue Nov 29 19:51:39 2016 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Sat Jan 14 19:00:09 2017 +0800
----------------------------------------------------------------------
.../storage/gtrecord/SequentialCubeTupleIterator.java | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4b00ec26/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index ee868c7..c621215 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -26,6 +26,7 @@ import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -33,6 +34,7 @@ import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.metadata.tuple.TupleInfo;
import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,6 +47,8 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
private static final Logger logger = LoggerFactory.getLogger(SequentialCubeTupleIterator.class);
+ private final int SCAN_THRESHOLD = KylinConfig.getInstanceFromEnv().getScanThreshold();
+
protected List<CubeSegmentScanner> scanners;
protected List<SegmentCubeTupleIterator> segmentCubeTupleIterators;
protected Iterator<ITuple> tupleIterator;
@@ -78,7 +82,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, returnTupleInfo)).getIterator();
}
}
-
+
public Comparator<ITuple> getTupleDimensionComparator(Cuboid cuboid, TupleInfo returnTupleInfo) {
// dimensionIndexOnTuple is for SQL with limit
List<Integer> temp = Lists.newArrayList();
@@ -92,7 +96,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
for (int i = 0; i < temp.size(); i++) {
dimensionIndexOnTuple[i] = temp.get(i);
}
-
+
return new Comparator<ITuple>() {
@Override
public int compare(ITuple o1, ITuple o2) {
@@ -137,7 +141,11 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
@Override
public ITuple next() {
- scanCount++;
+ // prevent the big query to make the Query Server OOM
+ if (scanCount++ > SCAN_THRESHOLD) {
+ throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " + SCAN_THRESHOLD);
+ }
+
if (++scanCountDelta >= 1000)
flushScanCountDelta();