[kylin] branch master updated: KYLIN-3950 Make optimize job algorithm configurable

[ni...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new ed266aa  KYLIN-3950 Make optimize job algorithm configurable
ed266aa is described below

commit ed266aa98d8524a344469b1e1ead8bfd462702d8
Author: chao long <wa...@qq.com>
AuthorDate: Fri Apr 12 15:04:42 2019 +0800

    KYLIN-3950 Make optimize job algorithm configurable
---
 .../src/main/java/org/apache/kylin/common/KylinConfigBase.java       | 4 ++++
 core-common/src/main/resources/kylin-defaults.properties             | 5 +++++
 .../org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java    | 2 +-
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index c99caeb..849f635 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -605,6 +605,10 @@ public abstract class KylinConfigBase implements Serializable {
         return Double.parseDouble(getOptional("kylin.cube.algorithm.layer-or-inmem-threshold", "7"));
     }
 
+    public boolean isAutoInmemToOptimize() {
+        return Boolean.parseBoolean(getOptional("kylin.cube.algorithm.inmem-auto-optimize", TRUE));
+    }
+
     public int getCubeAlgorithmAutoMapperLimit() {
         return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
     }
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index e0cb1f0..819c8af 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -207,6 +207,11 @@ kylin.cube.algorithm=layer
 # A smaller threshold prefers layer, a larger threshold prefers in-mem
 kylin.cube.algorithm.layer-or-inmem-threshold=7
 
+# auto use inmem algorithm:
+# 1, cube planner optimize job
+# 2, no source record
+kylin.cube.algorithm.inmem-auto-optimize=true
+
 kylin.cube.aggrgroup.max-combination=32768
 
 kylin.snapshot.max-mb=300
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
index 09db8e9..eabb1dc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
@@ -48,7 +48,7 @@ public class StatisticsDecisionUtil {
         KylinConfig kylinConf = seg.getConfig();
         String algPref = kylinConf.getCubeAlgorithm();
         CubingJob.AlgorithmEnum alg;
-        if (mapperOverlapRatio == 0) { // no source records
+        if (mapperOverlapRatio == 0 && kylinConf.isAutoInmemToOptimize()) { // no source records
             alg = CubingJob.AlgorithmEnum.INMEM;
         } else if (CubingJob.AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
             alg = CubingJob.AlgorithmEnum.INMEM;