You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/14 08:39:58 UTC

kylin git commit: KYLIN-1226 fix OOM when memory hungry measures are many in InMemCubeBuilder

Repository: kylin
Updated Branches:
  refs/heads/2.x-staging 9dc17f9b2 -> 22e03efc2


KYLIN-1226 fix OOM when memory hungry measures are many in InMemCubeBuilder


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22e03efc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22e03efc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22e03efc

Branch: refs/heads/2.x-staging
Commit: 22e03efc2bcd306ea1ddbcefa8e47246202b9858
Parents: 9dc17f9
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Dec 14 15:39:50 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Dec 14 15:39:50 2015 +0800

----------------------------------------------------------------------
 .../common/util/MemoryBudgetController.java     | 30 ++++++++++++--
 .../inmemcubing/AbstractInMemCubeBuilder.java   |  2 +-
 .../cube/inmemcubing/DoggedCubeBuilder.java     | 14 +++----
 .../cube/inmemcubing/InMemCubeBuilder.java      | 43 +++++++++-----------
 4 files changed, 55 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/22e03efc/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index dc49b17..e5a4fc7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -102,7 +102,7 @@ public class MemoryBudgetController {
             try {
                 reserve(consumer, requestMB);
                 if (debug && waitStart > 0)
-                    logger.debug(consumer + " waited " + (System.currentTimeMillis() - waitStart) + " ms on the " + requestMB + " MB request");
+                    logger.info(consumer + " waited " + (System.currentTimeMillis() - waitStart) + " ms on the " + requestMB + " MB request");
                 return;
             } catch (NotEnoughBudgetException ex) {
                 // retry
@@ -170,7 +170,7 @@ public class MemoryBudgetController {
 
         if (debug) {
             if (getSystemAvailMB() < getRemainingBudgetMB()) {
-                logger.debug("Remaining budget is " + getRemainingBudgetMB() + " MB free, but system only has " + getSystemAvailMB() + " MB free. If this persists, some memory calculation must be wrong.");
+                logger.info("Remaining budget is " + getRemainingBudgetMB() + " MB free, but system only has " + getSystemAvailMB() + " MB free. If this persists, some memory calculation must be wrong.");
             }
         }
     }
@@ -221,7 +221,7 @@ public class MemoryBudgetController {
             booking.remove(entry.consumer);
         }
         if (debug) {
-            logger.debug(entry.consumer + " reserved " + entry.reservedMB + " MB, total reserved " + totalReservedMB + " MB, remaining budget " + getRemainingBudgetMB() + " MB");
+            logger.info(entry.consumer + " reserved " + entry.reservedMB + " MB, total reserved " + totalReservedMB + " MB, remaining budget " + getRemainingBudgetMB() + " MB");
         }
 
         if (delta < 0) {
@@ -233,6 +233,30 @@ public class MemoryBudgetController {
         return true;
     }
 
+    public static int gcAndGetSystemAvailMB() {
+        final int tolerance = 1;
+        try {
+            int lastMB = -1;
+            while (true) {
+                Runtime.getRuntime().gc();
+                Thread.sleep(1000);
+                int thisMB = getSystemAvailMB();
+
+                if (lastMB < 0) {
+                    lastMB = thisMB;
+                    continue;
+                }
+                if (lastMB - thisMB < tolerance) {
+                    return thisMB;
+                }
+                lastMB = thisMB;
+            }
+        } catch (InterruptedException e) {
+            logger.error("", e);
+            return getSystemAvailMB();
+        }
+    }
+
     public static long getSystemAvailBytes() {
         Runtime runtime = Runtime.getRuntime();
         long totalMemory = runtime.totalMemory(); // current heap allocated to the VM process

http://git-wip-us.apache.org/repos/asf/kylin/blob/22e03efc/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index 58f94c1..15ab926 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -42,7 +42,7 @@ abstract public class AbstractInMemCubeBuilder {
     final protected Map<TblColRef, Dictionary<String>> dictionaryMap;
 
     protected int taskThreadCount = 4;
-    protected int reserveMemoryMB = 200;
+    protected int reserveMemoryMB = 100;
 
     public AbstractInMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
         if (cubeDesc == null)

http://git-wip-us.apache.org/repos/asf/kylin/blob/22e03efc/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index 5b6131f..ebc3b9b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -56,6 +56,10 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
 
     public DoggedCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
         super(cubeDesc, dictionaryMap);
+
+        // check memory more often if a single row is big
+        if (cubeDesc.hasMemoryHungryMeasures())
+            unitRows /= 10;
     }
 
     public void setSplitRowThreshold(int rowThreshold) {
@@ -75,7 +79,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
             int systemAvailMB = MemoryBudgetController.getSystemAvailMB();
 
             // InMemCubeBuilder will over-estimate base cuboid size by a factor, must cut ahead at least the same factor
-            cutAheadMB = (int) (systemAvailMB * InMemCubeBuilder.BASE_CUBOID_CACHE_OVERSIZE_FACTOR);
+            cutAheadMB = (int) (systemAvailMB * InMemCubeBuilder.getAggrCacheOversizeFactor(cubeDesc));
             logger.info("Cut ahead MB is " + cutAheadMB);
 
             int half = systemAvailMB / 2;
@@ -241,12 +245,8 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
                 }
 
                 // wait cuboid build done
-                while (last.isAlive()) {
-                    if (last.builder.isAllCuboidDone()) {
-                        break;
-                    }
-                    Thread.sleep(1000);
-                }
+                last.join();
+                
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
             }

http://git-wip-us.apache.org/repos/asf/kylin/blob/22e03efc/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 1e90407..7d6a829 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -60,7 +60,7 @@ import com.google.common.collect.Lists;
 public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
 
     private static Logger logger = LoggerFactory.getLogger(InMemCubeBuilder.class);
-    static final double BASE_CUBOID_CACHE_OVERSIZE_FACTOR = 0.15;
+    private static final double BASE_CUBOID_CACHE_OVERSIZE_FACTOR = 0.15;
 
     private final CuboidScheduler cuboidScheduler;
     private final long baseCuboidId;
@@ -292,21 +292,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     }
 
     private int getSystemAvailMB() {
-        // GC to be precise on memory left
-        Runtime.getRuntime().gc();
-        try {
-            Thread.sleep(1000);
-        } catch (InterruptedException e) {
-            logger.error("", e);
-        }
-        // GC again to be precise on memory left
-        Runtime.getRuntime().gc();
-        try {
-            Thread.sleep(1000);
-        } catch (InterruptedException e) {
-            logger.error("", e);
-        }
-        return MemoryBudgetController.getSystemAvailMB();
+        return MemoryBudgetController.gcAndGetSystemAvailMB();
     }
 
     private void makeMemoryBudget() {
@@ -327,20 +313,20 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
     }
 
     private CuboidResult createBaseCuboid(BlockingQueue<List<String>> input) throws IOException {
+        int mbBefore = getSystemAvailMB();
+        int mbAfter = 0;
+
+        long startTime = System.currentTimeMillis();
+        logger.info("Calculating base cuboid " + baseCuboidId + ", system avail " + mbBefore + " MB");
+
         GridTable baseCuboid = newGridTableByCuboidID(baseCuboidId);
         GTBuilder baseBuilder = baseCuboid.rebuild();
         IGTScanner baseInput = new InputConverter(baseCuboid.getInfo(), input);
 
-        int mbBefore = getSystemAvailMB();
-        int mbAfter = 0;
-
         Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
         GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
         GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true);
 
-        long startTime = System.currentTimeMillis();
-        logger.info("Calculating cuboid " + baseCuboidId);
-
         int count = 0;
         for (GTRecord r : aggregationScanner) {
             if (mbAfter == 0) {
@@ -357,13 +343,24 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
 
         int mbBaseAggrCacheOnHeap = mbAfter == 0 ? 0 : mbBefore - mbAfter;
         int mbEstimateBaseAggrCache = (int) (aggregationScanner.getEstimateSizeOfAggrCache() / MemoryBudgetController.ONE_MB);
-        int mbBaseAggrCache = (int) (mbBaseAggrCacheOnHeap * (1 + BASE_CUBOID_CACHE_OVERSIZE_FACTOR));
+        int mbBaseAggrCache = (int) (mbBaseAggrCacheOnHeap * (1 + getAggrCacheOversizeFactor(cubeDesc)));
         mbBaseAggrCache = Math.max(mbBaseAggrCache, 10); // let it be at least 10 MB
         logger.info("Base aggr cache is " + mbBaseAggrCache + " MB (heap " + mbBaseAggrCacheOnHeap + " MB, estimate " + mbEstimateBaseAggrCache + " MB)");
 
         return updateCuboidResult(baseCuboidId, baseCuboid, count, timeSpent, mbBaseAggrCache);
     }
 
+    // Aggregation cache need to be oversized such that spawned thread not only has memory for 
+    // aggregation cache but also has enough for temporary measures and others.
+    static double getAggrCacheOversizeFactor(CubeDesc cubeDesc) {
+        double r = BASE_CUBOID_CACHE_OVERSIZE_FACTOR;
+        for (MeasureDesc m : cubeDesc.getMeasures()) {
+            if (m.getFunction().getMeasureType().isMemoryHungry())
+                r += BASE_CUBOID_CACHE_OVERSIZE_FACTOR;
+        }
+        return r;
+    }
+
     private CuboidResult updateCuboidResult(long cuboidId, GridTable table, int nRows, long timeSpent, int aggrCacheMB) {
         if (aggrCacheMB <= 0) {
             aggrCacheMB = (int) Math.ceil(1.0 * nRows / baseResult.nRows * baseResult.aggrCacheMB);