You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/01/22 01:50:17 UTC
kylin git commit: KYLIN-3098 add a new config for maximum query row
count returned to user
Repository: kylin
Updated Branches:
refs/heads/master ab420496e -> f988401d4
KYLIN-3098 add a new config for maximum query row count returned to user
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f988401d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f988401d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f988401d
Branch: refs/heads/master
Commit: f988401d4821e0765105248baad90fd4e28e8fa2
Parents: ab42049
Author: Wang Ken <mi...@ebay.com>
Authored: Mon Dec 11 15:37:33 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun Jan 21 21:27:28 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../src/main/java/org/apache/kylin/common/QueryContext.java | 9 +++++++++
.../kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 6 ++++++
3 files changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f988401d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 341ca3f..cfe6fe2 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1225,6 +1225,10 @@ abstract public class KylinConfigBase implements Serializable {
return value > 0 ? value : Long.MAX_VALUE;
}
+ public long getQueryMaxReturnRows() {
+ return Integer.parseInt(this.getOptional("kylin.query.max-return-rows", "5000000"));
+ }
+
public int getTranslatedInClauseMaxSize() {
return Integer.parseInt(getOptional("kylin.query.translated-in-clause-max-size", String.valueOf(1024 * 1024)));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f988401d/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 6703831..cb1b09c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -51,6 +51,7 @@ public class QueryContext {
private String username;
private Set<String> groups;
private AtomicLong scannedRows = new AtomicLong();
+ private AtomicLong returnedRows = new AtomicLong();
private AtomicLong scannedBytes = new AtomicLong();
private AtomicBoolean isRunning = new AtomicBoolean(true);
@@ -128,6 +129,14 @@ public class QueryContext {
return scannedRows.addAndGet(deltaRows);
}
+ public long getReturnedRows() {
+ return returnedRows.get();
+ }
+
+ public long addAndGetReturnedRows(long deltaRows) {
+ return returnedRows.addAndGet(deltaRows);
+ }
+
public long getScannedBytes() {
return scannedBytes.get();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f988401d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index ddf62b7..16b8db2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -286,6 +286,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
Stats stats = result.getStats();
queryContext.addAndGetScannedRows(stats.getScannedRowCount());
queryContext.addAndGetScannedBytes(stats.getScannedBytes());
+ queryContext.addAndGetReturnedRows(stats.getScannedRowCount()
+ - stats.getAggregatedRowCount() - stats.getFilteredRowCount());
RuntimeException rpcException = null;
if (result.getStats().getNormalComplete() != 1) {
@@ -305,6 +307,10 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
rpcException = new ResourceLimitExceededException(
"Query scanned " + queryContext.getScannedBytes() + " bytes exceeds threshold "
+ cubeSeg.getConfig().getQueryMaxScanBytes());
+ } else if (queryContext.getReturnedRows() > cubeSeg.getConfig().getQueryMaxReturnRows()) {
+ rpcException = new ResourceLimitExceededException(
+ "Query returned " + queryContext.getReturnedRows() + " rows exceeds threshold "
+ + cubeSeg.getConfig().getQueryMaxReturnRows());
}
if (rpcException != null) {