You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2019/03/14 06:10:42 UTC
[kylin] branch master updated: KYLIN-3866 Whether to set
mapreduce.application.classpath is determined by the user
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new d23d2bd KYLIN-3866 Whether to set mapreduce.application.classpath is determined by the user
d23d2bd is described below
commit d23d2bddab3263d39059492b91f71d8091198057
Author: Guangxu Cheng <gx...@apache.org>
AuthorDate: Mon Mar 11 17:37:06 2019 +0800
KYLIN-3866 Whether to set mapreduce.application.classpath is determined by the user
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../java/org/apache/kylin/common/KylinConfigBase.java | 5 +++++
.../kylin/engine/mr/common/AbstractHadoopJob.java | 17 ++++++++++-------
2 files changed, 15 insertions(+), 7 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3edd458..82f79eb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1332,6 +1332,11 @@ public abstract class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "10"));
}
+
+ public boolean isUseLocalClasspathEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.engine.mr.use-local-classpath", TRUE));
+ }
+
// ============================================================================
// ENGINE.SPARK
// ============================================================================
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index ca92dfd..76286e3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -214,15 +214,18 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
String kylinKafkaDependency = System.getProperty("kylin.kafka.dependency");
Configuration jobConf = job.getConfiguration();
- String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
- if (classpath == null || classpath.length() == 0) {
- logger.info("Didn't find " + MAP_REDUCE_CLASSPATH
+
+ if (kylinConf.isUseLocalClasspathEnabled()) {
+ String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
+ if (classpath == null || classpath.length() == 0) {
+ logger.info("Didn't find " + MAP_REDUCE_CLASSPATH
+ " in job configuration, will run 'mapred classpath' to get the default value.");
- classpath = getDefaultMapRedClasspath();
- logger.info("The default mapred classpath is: " + classpath);
- }
+ classpath = getDefaultMapRedClasspath();
+ logger.info("The default mapred classpath is: " + classpath);
+ }
- jobConf.set(MAP_REDUCE_CLASSPATH, classpath);
+ jobConf.set(MAP_REDUCE_CLASSPATH, classpath);
+ }
logger.trace("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH));
/*