You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/14 08:21:03 UTC
kylin git commit: KYLIN-1226 A better wait GC done
Repository: kylin
Updated Branches:
refs/heads/2.0-rc 0ba1e065c -> 183b31a66
KYLIN-1226 A better wait GC done
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/183b31a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/183b31a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/183b31a6
Branch: refs/heads/2.0-rc
Commit: 183b31a664e4a79ca889462a5d8e4d5d69d7dc5a
Parents: 0ba1e06
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Dec 14 15:20:19 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Dec 14 15:20:49 2015 +0800
----------------------------------------------------------------------
.../common/util/MemoryBudgetController.java | 24 +++++++++++++++++
.../cube/inmemcubing/InMemCubeBuilder.java | 28 +++++---------------
2 files changed, 31 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/183b31a6/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index e454414..d94f32b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -233,6 +233,30 @@ public class MemoryBudgetController {
return true;
}
+ public static int gcAndGetSystemAvailMB() {
+ final int tolerance = 1;
+ try {
+ int lastMB = -1;
+ while (true) {
+ Runtime.getRuntime().gc();
+ Thread.sleep(1000);
+ int thisMB = getSystemAvailMB();
+
+ if (lastMB < 0) {
+ lastMB = thisMB;
+ continue;
+ }
+ if (lastMB - thisMB < tolerance) {
+ return thisMB;
+ }
+ lastMB = thisMB;
+ }
+ } catch (InterruptedException e) {
+ logger.error("", e);
+ return getSystemAvailMB();
+ }
+ }
+
public static long getSystemAvailBytes() {
Runtime runtime = Runtime.getRuntime();
long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process
http://git-wip-us.apache.org/repos/asf/kylin/blob/183b31a6/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index a3e3a03..97c96fb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -295,21 +295,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
private int getSystemAvailMB() {
- // GC to be precise on memory left
- Runtime.getRuntime().gc();
- try {
- Thread.sleep(2500);
- } catch (InterruptedException e) {
- logger.error("", e);
- }
- // GC again to be precise on memory left
- Runtime.getRuntime().gc();
- try {
- Thread.sleep(2500);
- } catch (InterruptedException e) {
- logger.error("", e);
- }
- return MemoryBudgetController.getSystemAvailMB();
+ return MemoryBudgetController.gcAndGetSystemAvailMB();
}
private void makeMemoryBudget() {
@@ -330,20 +316,20 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
private CuboidResult createBaseCuboid(BlockingQueue<List<String>> input) throws IOException {
+ int mbBefore = getSystemAvailMB();
+ int mbAfter = 0;
+
+ long startTime = System.currentTimeMillis();
+ logger.info("Calculating base cuboid " + baseCuboidId + ", system avail " + mbBefore + " MB");
+
GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId);
GTBuilder baseBuilder = baseCuboid.rebuild();
IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input);
- int mbBefore = getSystemAvailMB();
- int mbAfter = 0;
-
Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true);
- long startTime = System.currentTimeMillis();
- logger.info("Calculating base cuboid " + baseCuboidId + ", system avail " + mbBefore + " MB");
-
int count = 0;
for (GTRecord r : aggregationScanner) {
if (mbAfter == 0) {