You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/14 08:39:58 UTC
kylin git commit: KYLIN-1226 fix OOM when memory hungry measures are
many in InMemCubeBuilder
Repository: kylin
Updated Branches:
refs/heads/2.x-staging 9dc17f9b2 -> 22e03efc2
KYLIN-1226 fix OOM when memory hungry measures are many in InMemCubeBuilder
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22e03efc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22e03efc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22e03efc
Branch: refs/heads/2.x-staging
Commit: 22e03efc2bcd306ea1ddbcefa8e47246202b9858
Parents: 9dc17f9
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Dec 14 15:39:50 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Dec 14 15:39:50 2015 +0800
----------------------------------------------------------------------
.../common/util/MemoryBudgetController.java | 30 ++++++++++++--
.../inmemcubing/AbstractInMemCubeBuilder.java | 2 +-
.../cube/inmemcubing/DoggedCubeBuilder.java | 14 +++----
.../cube/inmemcubing/InMemCubeBuilder.java | 43 +++++++++-----------
4 files changed, 55 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/22e03efc/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index dc49b17..e5a4fc7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -102,7 +102,7 @@ public class MemoryBudgetController {
try {
reserve(consumer, requestMB);
if (debug && waitStart > 0)
- logger.debug(consumer + " waited " + (System.currentTimeMillis() - waitStart) + " ms on the " + requestMB + " MB request");
+ logger.info(consumer + " waited " + (System.currentTimeMillis() - waitStart) + " ms on the " + requestMB + " MB request");
return;
} catch (NotEnoughBudgetException ex) {
// retry
@@ -170,7 +170,7 @@ public class MemoryBudgetController {
if (debug) {
if (getSystemAvailMB() < getRemainingBudgetMB()) {
- logger.debug("Remaining budget is " + getRemainingBudgetMB() + " MB free, but system only has " + getSystemAvailMB() + " MB free. If this persists, some memory calculation must be wrong.");
+ logger.info("Remaining budget is " + getRemainingBudgetMB() + " MB free, but system only has " + getSystemAvailMB() + " MB free. If this persists, some memory calculation must be wrong.");
}
}
}
@@ -221,7 +221,7 @@ public class MemoryBudgetController {
booking.remove(entry.consumer);
}
if (debug) {
- logger.debug(entry.consumer + " reserved " + entry.reservedMB + " MB, total reserved " + totalReservedMB + " MB, remaining budget " + getRemainingBudgetMB() + " MB");
+ logger.info(entry.consumer + " reserved " + entry.reservedMB + " MB, total reserved " + totalReservedMB + " MB, remaining budget " + getRemainingBudgetMB() + " MB");
}
if (delta < 0) {
@@ -233,6 +233,30 @@ public class MemoryBudgetController {
return true;
}
+ public static int gcAndGetSystemAvailMB() {
+ final int tolerance = 1;
+ try {
+ int lastMB = -1;
+ while (true) {
+ Runtime.getRuntime().gc();
+ Thread.sleep(1000);
+ int thisMB = getSystemAvailMB();
+
+ if (lastMB < 0) {
+ lastMB = thisMB;
+ continue;
+ }
+ if (lastMB - thisMB < tolerance) {
+ return thisMB;
+ }
+ lastMB = thisMB;
+ }
+ } catch (InterruptedException e) {
+ logger.error("", e);
+ return getSystemAvailMB();
+ }
+ }
+
public static long getSystemAvailBytes() {
Runtime runtime = Runtime.getRuntime();
long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process
http://git-wip-us.apache.org/repos/asf/kylin/blob/22e03efc/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 58f94c1..15ab926 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -42,7 +42,7 @@ abstract public class AbstractInMemCubeBuilder {
final protected Map<TblColRef, Dictionary<String>> dictionaryMap;
protected int taskThreadCount = 4;
- protected int reserveMemoryMB = 200;
+ protected int reserveMemoryMB = 100;
public AbstractInMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
if (cubeDesc == null)
http://git-wip-us.apache.org/repos/asf/kylin/blob/22e03efc/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index 5b6131f..ebc3b9b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -56,6 +56,10 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
public DoggedCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
super(cubeDesc, dictionaryMap);
+
+ // check memory more often if a single row is big
+ if (cubeDesc.hasMemoryHungryMeasures())
+ unitRows /= 10;
}
public void setSplitRowThreshold(int rowThreshold) {
@@ -75,7 +79,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
int systemAvailMB = MemoryBudgetController.getSystemAvailMB();
// InMemCubeBuilder will over-estimate base cuboid size by a factor, must cut ahead at least the same factor
- cutAheadMB = (int) (systemAvailMB * InMemCubeBuilder.BASE_CUBOID_CACHE_OVERSIZE_FACTOR);
+ cutAheadMB = (int) (systemAvailMB * InMemCubeBuilder.getAggrCacheOversizeFactor(cubeDesc));
logger.info("Cut ahead MB is " + cutAheadMB);
int half = systemAvailMB / 2;
@@ -241,12 +245,8 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
}
// wait cuboid build done
- while (last.isAlive()) {
- if (last.builder.isAllCuboidDone()) {
- break;
- }
- Thread.sleep(1000);
- }
+ last.join();
+
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/22e03efc/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 1e90407..7d6a829 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -60,7 +60,7 @@ import com.google.common.collect.Lists;
public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private static Logger logger = LoggerFactory.getLogger(InMemCubeBuilder.class);
- static final double BASE_CUBOID_CACHE_OVERSIZE_FACTOR = 0.15;
+ private static final double BASE_CUBOID_CACHE_OVERSIZE_FACTOR = 0.15;
private final CuboidScheduler cuboidScheduler;
private final long baseCuboidId;
@@ -292,21 +292,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
private int getSystemAvailMB() {
- // GC to be precise on memory left
- Runtime.getRuntime().gc();
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- logger.error("", e);
- }
- // GC again to be precise on memory left
- Runtime.getRuntime().gc();
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- logger.error("", e);
- }
- return MemoryBudgetController.getSystemAvailMB();
+ return MemoryBudgetController.gcAndGetSystemAvailMB();
}
private void makeMemoryBudget() {
@@ -327,20 +313,20 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
}
private CuboidResult createBaseCuboid(BlockingQueue<List<String>> input) throws IOException {
+ int mbBefore = getSystemAvailMB();
+ int mbAfter = 0;
+
+ long startTime = System.currentTimeMillis();
+ logger.info("Calculating base cuboid " + baseCuboidId + ", system avail " + mbBefore + " MB");
+
GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId);
GTBuilder baseBuilder = baseCuboid.rebuild();
IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input);
- int mbBefore = getSystemAvailMB();
- int mbAfter = 0;
-
Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true);
- long startTime = System.currentTimeMillis();
- logger.info("Calculating cuboid " + baseCuboidId);
-
int count = 0;
for (GTRecord r : aggregationScanner) {
if (mbAfter == 0) {
@@ -357,13 +343,24 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
int mbBaseAggrCacheOnHeap = mbAfter == 0 ? 0 : mbBefore - mbAfter;
int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB);
- int mbBaseAggrCache = (int) (mbBaseAggrCacheOnHeap * (1 + BASE_CUBOID_CACHE_OVERSIZE_FACTOR));
+ int mbBaseAggrCache = (int) (mbBaseAggrCacheOnHeap * (1 + getAggrCacheOversizeFactor(cubeDesc)));
mbBaseAggrCache = Math.max(mbBaseAggrCache, 10); // let it be at least 10 MB
logger.info("Base aggr cache is " + mbBaseAggrCache + " MB (heap " + mbBaseAggrCacheOnHeap + " MB, estimate " + mbEstimateBaseAggrCache + " MB)");
return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, mbBaseAggrCache);
}
+ // Aggregation cache need to be oversized such that spawned thread not only has memory for
+ // aggregation cache but also has enough for temporary measures and others.
+ static double getAggrCacheOversizeFactor(CubeDesc cubeDesc) {
+ double r = BASE_CUBOID_CACHE_OVERSIZE_FACTOR;
+ for (MeasureDesc m : cubeDesc.getMeasures()) {
+ if (m.getFunction().getMeasureType().isMemoryHungry())
+ r += BASE_CUBOID_CACHE_OVERSIZE_FACTOR;
+ }
+ return r;
+ }
+
private CuboidResult updateCuboidResult(long cuboidId, GridTable table, int nRows, long timeSpent, int aggrCacheMB) {
if (aggrCacheMB <= 0) {
aggrCacheMB = (int) Math.ceil(1.0 * nRows / baseResult.nRows * baseResult.aggrCacheMB);