You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/02 11:48:33 UTC
incubator-kylin git commit: KYLIN-942 only check mem use at query time
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging 2f865b80e -> 101ef86ac
KYLIN-942 only check mem use at query time
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/101ef86a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/101ef86a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/101ef86a
Branch: refs/heads/2.x-staging
Commit: 101ef86ac8a411d290d4bde80eeb29d91afcfa95
Parents: 2f865b8
Author: honma <ho...@ebay.com>
Authored: Mon Nov 2 18:52:36 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Nov 2 18:52:58 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/MemoryBudgetController.java | 2 +-
.../org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java | 2 +-
.../java/org/apache/kylin/gridtable/GTAggregateScanner.java | 5 +++--
.../src/main/java/org/apache/kylin/gridtable/GTScanRequest.java | 2 +-
4 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index 2c26666..dc49b17 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -57,7 +57,7 @@ public class MemoryBudgetController {
public static final MemoryBudgetController ZERO_BUDGET = new MemoryBudgetController(0);
public static final int ONE_MB = 1024 * 1024;
- public static final int ONE_GB = 1024 * 1024 * 1024;
+ public static final long ONE_GB = 1024 * 1024 * 1024;
private static final Logger logger = LoggerFactory.getLogger(MemoryBudgetController.class);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index d7aa50a..a179d70 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -343,7 +343,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
GTScanRequest req = new GTScanRequest(baseCuboid.getInfo(), null, dimensionMetricsBitSet.getFirst(), dimensionMetricsBitSet.getSecond(), metricsAggrFuncs, null);
- GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, false);
+ GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, true);
long startTime = System.currentTimeMillis();
logger.info("Calculating cuboid " + baseCuboidId);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 35c1210..cd2881e 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -142,8 +142,9 @@ public class GTAggregateScanner implements IGTScanner {
void aggregate(GTRecord r) {
if (enableMemCheck && (++aggregatedRowCount % 100000 == 0)) {
- if (estimatedMemSize() > MemoryBudgetController.ONE_GB) {
- throw new RuntimeException("AggregationCache exceed 1GB");
+ long estimated = estimatedMemSize();
+ if (estimated > 10 * MemoryBudgetController.ONE_GB) {
+ throw new RuntimeException("AggregationCache exceed 10GB, estimated size is: " + estimated);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/101ef86a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index ba997ec..2c284c9 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -118,7 +118,7 @@ public class GTScanRequest {
}
public IGTScanner decorateScanner(IGTScanner scanner) throws IOException {
- return decorateScanner(scanner, true, true, true);
+ return decorateScanner(scanner, true, true, false);//by default do not check mem
}
/**