You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/09/25 05:53:46 UTC
hadoop git commit: YARN-8817. [Submarine] In cases when user doesn't
ask HDFS path while submitting job but framework requires user to set HDFS
related environments. Contributed by Wangda Tan.
Repository: hadoop
Updated Branches:
refs/heads/trunk 9c3fbbc4f -> d0bc6a27f
YARN-8817. [Submarine] In cases when user doesn't ask HDFS path while submitting job but framework requires user to set HDFS related environments. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0bc6a27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0bc6a27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0bc6a27
Branch: refs/heads/trunk
Commit: d0bc6a27fdbeea2e06c0151e72347946ababf2e4
Parents: 9c3fbbc
Author: Sunil G <su...@apache.org>
Authored: Tue Sep 25 11:23:34 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Sep 25 11:23:34 2018 +0530
----------------------------------------------------------------------
.../yarnservice/YarnServiceJobSubmitter.java | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bc6a27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index 5855287..d57c675 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -102,6 +102,13 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
}
}
+ private boolean needHdfs(String content) {
+ if (content != null && content.contains("hdfs://")) {
+ return true;
+ }
+ return false;
+ }
+
private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
PrintWriter fw, Component comp) throws IOException {
// Find envs to use HDFS
@@ -121,14 +128,11 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
boolean lackingEnvs = false;
- if ((parameters.getInputPath() != null && parameters.getInputPath()
- .contains("hdfs://")) || (parameters.getCheckpointPath() != null
- && parameters.getCheckpointPath().contains("hdfs://")) || (
- parameters.getSavedModelPath() != null && parameters.getSavedModelPath()
- .contains("hdfs://")) || hadoopEnv) {
+ if (needHdfs(parameters.getInputPath()) || needHdfs(
+ parameters.getPSLaunchCmd()) || needHdfs(
+ parameters.getWorkerLaunchCmd()) || hadoopEnv) {
// HDFS is asked either in input or output, set LD_LIBRARY_PATH
// and classpath
-
if (hdfsHome != null) {
// Unset HADOOP_HOME/HADOOP_YARN_HOME to make sure host machine's envs
// won't pollute docker's env.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org