You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/12/14 06:59:23 UTC
[2/3] kylin git commit: KYLIN-2993 Add special mr config for base
cuboid step
KYLIN-2993 Add special mr config for base cuboid step
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad3dd3f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad3dd3f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad3dd3f6
Branch: refs/heads/master
Commit: ad3dd3f63d399640142bd2c97669080bc3e37560
Parents: a79035a
Author: kangkaisen <ka...@meituan.com>
Authored: Mon Sep 4 10:29:00 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 14 14:58:17 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../kylin/storage/hbase/steps/HBaseMROutput2Transition.java | 9 +++++++++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad3dd3f6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d59c98d..524b1d4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1003,6 +1003,10 @@ abstract public class KylinConfigBase implements Serializable {
return getPropertiesByPrefix("kylin.engine.mr.uhc-config-override.");
}
+ public Map<String, String> getBaseCuboidMRConfigOverride() {
+ return getPropertiesByPrefix("kylin.engine.mr.base-cuboid-config-override.");
+ }
+
public Map<String, String> getSparkConfigOverride() {
return getPropertiesByPrefix("kylin.engine.spark-conf.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad3dd3f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index db3f7f4..8817cb2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -19,6 +19,7 @@
package org.apache.kylin.storage.hbase.steps;
import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -99,6 +100,14 @@ public class HBaseMROutput2Transition implements IMROutput2 {
int level) throws Exception {
int reducerNum = 1;
Class mapperClass = job.getMapperClass();
+
+ //allow user specially set config for base cuboid step
+ if (mapperClass == HiveToBaseCuboidMapper.class) {
+ for (Map.Entry<String, String> entry : segment.getConfig().getBaseCuboidMRConfigOverride().entrySet()) {
+ job.getConfiguration().set(entry.getKey(), entry.getValue());
+ }
+ }
+
if (mapperClass == HiveToBaseCuboidMapper.class || mapperClass == NDCuboidMapper.class) {
reducerNum = MapReduceUtil.getLayeredCubingReduceTaskNum(segment, cuboidScheduler,
AbstractHadoopJob.getTotalMapInputMB(job), level);