You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/14 23:59:20 UTC
kylin git commit: KYLIN-1226 A better default of reserved memory
Repository: kylin
Updated Branches:
refs/heads/2.0-rc 949efc558 -> 9c450617b
KYLIN-1226 A better default of reserved memory
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9c450617
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9c450617
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9c450617
Branch: refs/heads/2.0-rc
Commit: 9c450617b2ee245e7a3354f89de166fb2fd6eafb
Parents: 949efc5
Author: Yang Li <li...@apache.org>
Authored: Tue Dec 15 06:59:10 2015 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Dec 15 06:59:10 2015 +0800
----------------------------------------------------------------------
.../cube/inmemcubing/DoggedCubeBuilder.java | 24 ++++++++++++--------
1 file changed, 15 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9c450617/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index 36a45e1..e87d8ff 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -56,10 +56,23 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
public DoggedCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<?>> dictionaryMap) {
super(cubeDesc, dictionaryMap);
-
+
// check memory more often if a single row is big
if (cubeDesc.hasMemoryHungryCountDistinctMeasures())
unitRows /= 10;
+
+ // a meaningful default reserved memory
+ int sysAvailMB = MemoryBudgetController.getSystemAvailMB();
+ int reserve = Math.max(sysAvailMB / 10, 100); // 10% of system available, or 100 MB at least
+ int halfSysAvail = sysAvailMB / 2;
+ if (reserve > halfSysAvail) { // but no more than half of system available
+ logger.info("Reserve " + reserve + " MB is more than half of system avail " + sysAvailMB + " MB, override to " + halfSysAvail);
+ reserve = halfSysAvail;
+ }
+
+ setReserveMemoryMB(reserve);
+ logger.info("Reserve memory is set to " + reserve + " MB");
+
}
public void setSplitRowThreshold(int rowThreshold) {
@@ -75,13 +88,6 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
private class BuildOnce {
BuildOnce() {
- int systemAvailMB = MemoryBudgetController.getSystemAvailMB();
-
- int half = systemAvailMB / 2;
- if (getReserveMemoryMB() > half) {
- logger.info("Reserve " + getReserveMemoryMB() + " MB is more than half of system avail " + systemAvailMB + " MB, override to " + half);
- setReserveMemoryMB(half);
- }
}
public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException {
@@ -241,7 +247,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
// wait cuboid build done
last.join();
-
+
} catch (InterruptedException e) {
throw new RuntimeException(e);
}