You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2020/03/26 06:52:16 UTC
[kylin] branch master updated: KYLIN-4379: use config overwrite for
all MR job
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new df26d77 KYLIN-4379: use config overwrite for all MR job
df26d77 is described below
commit df26d77d2007ad7a64b639c3c1047fc81c28de86
Author: Zhou Kang <zh...@xiaomi.com>
AuthorDate: Mon Feb 17 13:40:36 2020 +0800
KYLIN-4379: use config overwrite for all MR job
---
.../engine/mr/common/MapReduceExecutable.java | 26 +++++++++++++---------
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
index 260c9ee..a33f171 100755
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/MapReduceExecutable.java
@@ -489,18 +489,24 @@ public class MapReduceExecutable extends AbstractExecutable {
conf.addResource(new Path(confFile));
}
- if (StringUtils.isNotBlank(cubeName)) {
- for (Map.Entry<String, String> entry : CubeManager.getInstance(config).getCube(cubeName).getConfig()
- .getMRConfigOverride().entrySet()) {
+ KylinConfig configOverride;
+ if (cubeName != null) {
+ configOverride = CubeManager.getInstance(config).getCube(cubeName).getConfig();
+ } else {
+ configOverride = config;
+ }
+
+ for (Map.Entry<String, String> entry : configOverride.getMRConfigOverride().entrySet()) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
+ if (conf.get("mapreduce.job.is-mem-hungry") != null
+ && Boolean.parseBoolean(conf.get("mapreduce.job.is-mem-hungry"))) {
+ for (Map.Entry<String, String> entry : configOverride.getMemHungryConfigOverride().entrySet()) {
conf.set(entry.getKey(), entry.getValue());
}
- if (conf.get("mapreduce.job.is-mem-hungry") != null
- && Boolean.parseBoolean(conf.get("mapreduce.job.is-mem-hungry"))) {
- for (Map.Entry<String, String> entry : CubeManager.getInstance(config).getCube(cubeName).getConfig()
- .getMemHungryConfigOverride().entrySet()) {
- conf.set(entry.getKey(), entry.getValue());
- }
- }
+ }
+
+ if (StringUtils.isNotBlank(cubeName)) {
remainingArgs.add("-" + BatchConstants.ARG_CUBE_NAME);
remainingArgs.add(cubeName);
}