You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/12/04 09:23:22 UTC
[09/10] kylin git commit: KYLIN-1190 Make memory budget per query
configurable
KYLIN-1190 Make memory budget per query configurable
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e8e2d27a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e8e2d27a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e8e2d27a
Branch: refs/heads/2.x-staging
Commit: e8e2d27ab82d6565615a27d4a441f019a5b5aafa
Parents: 6b20569
Author: lidongsjtu <do...@ebay.com>
Authored: Thu Dec 3 09:50:19 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:29:07 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfig.java | 4 ++++
.../kylin/storage/hbase/cube/v1/CubeStorageQuery.java | 12 ++++++++----
.../kylin/storage/hbase/cube/v2/CubeStorageQuery.java | 11 +++++++----
3 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index b08df5f..d2a94c5 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -492,6 +492,10 @@ public class KylinConfig implements Serializable {
return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024)));
}
+ public long getQueryMemBudget() {
+ return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024)));
+ }
+
public boolean isQuerySecureEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 4d34943..2fa0490 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -82,7 +82,6 @@ public class CubeStorageQuery implements ICachableStorageQuery {
private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class);
private static final int MERGE_KEYRANGE_THRESHOLD = 100;
- private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G
private final CubeInstance cubeInstance;
private final CubeDesc cubeDesc;
@@ -732,8 +731,13 @@ public class CubeStorageQuery implements ICachableStorageQuery {
}
}
- long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
- context.setThreshold((int) rowEst);
+ long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst;
+ if (rowEst > 0) {
+ logger.info("Memory budget is set to: " + rowEst);
+ context.setThreshold((int) rowEst);
+ } else {
+ logger.info("Memory budget is not set.");
+ }
}
private void setLimit(TupleFilter filter, StorageContext context) {
@@ -760,7 +764,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
topnLiteralCol = func.getTopNLiteralColumn();
}
}
-
+
// if TopN is not involved
if (topnFunc == null)
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e8e2d27a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 0c8c3bd..59ed99b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -41,8 +41,6 @@ public class CubeStorageQuery implements ICachableStorageQuery {
private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class);
- private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G
-
private final CubeInstance cubeInstance;
private final CubeDesc cubeDesc;
@@ -362,8 +360,13 @@ public class CubeStorageQuery implements ICachableStorageQuery {
rowSizeEst += func.getReturnDataType().getStorageBytesEstimate();
}
- long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
- context.setThreshold((int) rowEst);
+ long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst;
+ if (rowEst > 0) {
+ logger.info("Memory budget is set to: " + rowEst);
+ context.setThreshold((int) rowEst);
+ } else {
+ logger.info("Memory budget is not set.");
+ }
}
private void setLimit(TupleFilter filter, StorageContext context) {