You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/03/18 18:44:21 UTC
git commit: TEZ-939. Fix build break caused by changes in YARN-1824.
(sseth)
Repository: incubator-tez
Updated Branches:
refs/heads/master ae6c9746c -> 4a60f05c1
TEZ-939. Fix build break caused by changes in YARN-1824. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/4a60f05c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/4a60f05c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/4a60f05c
Branch: refs/heads/master
Commit: 4a60f05c105188aefb48f64c4ccf2efbc4b26da0
Parents: ae6c974
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Mar 18 10:43:30 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Mar 18 10:43:30 2014 -0700
----------------------------------------------------------------------
.../org/apache/tez/client/TezClientUtils.java | 20 +++---
.../java/org/apache/tez/client/TezSession.java | 11 +--
.../org/apache/tez/common/TezYARNUtils.java | 72 ++++++++++++++++++++
.../apache/tez/mapreduce/hadoop/MRHelpers.java | 26 +++----
4 files changed, 102 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4a60f05c/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index ff45cf5..d47ff80 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -70,10 +70,10 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
-import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
+import org.apache.tez.common.TezYARNUtils;
import org.apache.tez.common.impl.LogUtils;
import org.apache.tez.common.security.TokenCache;
import org.apache.tez.dag.api.DAG;
@@ -401,7 +401,8 @@ public class TezClientUtils {
if (amConfig.getEnv() != null) {
for (Map.Entry<String, String> entry : amConfig.getEnv().entrySet()) {
- Apps.addToEnvironment(environment, entry.getKey(), entry.getValue());
+ TezYARNUtils.addToEnvironment(environment, entry.getKey(), entry.getValue(),
+ File.pathSeparator);
}
}
@@ -567,20 +568,22 @@ public class TezClientUtils {
static String getFrameworkClasspath(Configuration conf) {
Map<String, String> environment = new HashMap<String, String>();
- Apps.addToEnvironment(environment,
+ TezYARNUtils.addToEnvironment(environment,
Environment.CLASSPATH.name(),
- Environment.PWD.$());
+ Environment.PWD.$(),
+ File.pathSeparator);
- Apps.addToEnvironment(environment,
+ TezYARNUtils.addToEnvironment(environment,
Environment.CLASSPATH.name(),
- Environment.PWD.$() + File.separator + "*");
+ Environment.PWD.$() + File.separator + "*",
+ File.pathSeparator);
// Add YARN/COMMON/HDFS jars and conf locations to path
for (String c : conf.getStrings(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
- Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),
- c.trim());
+ TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
+ c.trim(), File.pathSeparator);
}
return environment.get(Environment.CLASSPATH.name());
}
@@ -749,5 +752,4 @@ public class TezClientUtils {
}
return proxy;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4a60f05c/tez-api/src/main/java/org/apache/tez/client/TezSession.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezSession.java b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
index 6ea80a9..f16e509 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezSession.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezSession.java
@@ -18,6 +18,7 @@
package org.apache.tez.client;
+import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -33,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Apps;
+import org.apache.tez.common.TezYARNUtils;
import org.apache.tez.dag.api.DAG;
import org.apache.tez.dag.api.DAGSubmissionTimedOut;
import org.apache.tez.dag.api.DagTypeConverters;
@@ -150,9 +151,9 @@ public class TezSession {
.getFrameworkClasspath(sessionConfig.getYarnConfiguration());
for (Vertex v : dag.getVertices()) {
Map<String, String> taskEnv = v.getTaskEnvironment();
- Apps.addToEnvironment(taskEnv,
+ TezYARNUtils.addToEnvironment(taskEnv,
ApplicationConstants.Environment.CLASSPATH.name(),
- classpath);
+ classpath, File.pathSeparator);
}
DAGPlan dagPlan = dag.createDag(sessionConfig.getTezConfiguration());
@@ -295,9 +296,9 @@ public class TezSession {
String classpath = TezClientUtils
.getFrameworkClasspath(sessionConfig.getYarnConfiguration());
Map<String, String> contextEnv = context.getEnvironment();
- Apps.addToEnvironment(contextEnv,
+ TezYARNUtils.addToEnvironment(contextEnv,
ApplicationConstants.Environment.CLASSPATH.name(),
- classpath);
+ classpath, File.pathSeparator);
DAGClientAMProtocolRPC.PreWarmRequestProto.Builder
preWarmReqProtoBuilder =
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4a60f05c/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
new file mode 100644
index 0000000..aceec81
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.tez.common;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringInterner;
+
+@Private
+public class TezYARNUtils {
+
+ public static void setEnvFromInputString(Map<String, String> env,
+ String envString, String classPathSeparator) {
+ if (envString != null && envString.length() > 0) {
+ String childEnvs[] = envString.split(",");
+ Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex());
+ for (String cEnv : childEnvs) {
+ String[] parts = cEnv.split("="); // split on '='
+ Matcher m = p.matcher(parts[1]);
+ StringBuffer sb = new StringBuffer();
+ while (m.find()) {
+ String var = m.group(1);
+ // replace $env with the child's env constructed by tt's
+ String replace = env.get(var);
+ // if this key is not configured by the tt for the child .. get it
+ // from the tt's env
+ if (replace == null)
+ replace = System.getenv(var);
+ // the env key is note present anywhere .. simply set it
+ if (replace == null)
+ replace = "";
+ m.appendReplacement(sb, Matcher.quoteReplacement(replace));
+ }
+ m.appendTail(sb);
+ addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
+ }
+ }
+ }
+
+ public static void addToEnvironment(
+ Map<String, String> environment,
+ String variable, String value, String classPathSeparator) {
+ String val = environment.get(variable);
+ if (val == null) {
+ val = value;
+ } else {
+ val = val + classPathSeparator + value;
+ }
+ environment.put(StringInterner.weakIntern(variable),
+ StringInterner.weakIntern(val));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/4a60f05c/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 91fa4ae..3ed3702 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -19,6 +19,7 @@
package org.apache.tez.mapreduce.hadoop;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
@@ -49,7 +50,6 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
import org.apache.hadoop.mapreduce.split.JobSplitWriter;
import org.apache.hadoop.mapreduce.v2.proto.MRProtos;
-import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -59,10 +59,10 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.TezYARNUtils;
import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.TezConfiguration;
@@ -819,19 +819,20 @@ public class MRHelpers {
MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));
// Add pwd to LD_LIBRARY_PATH, add this before adding anything else
- Apps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
- Environment.PWD.$());
+ TezYARNUtils.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
+ Environment.PWD.$(), File.pathSeparator);
// Add the env variables passed by the admin
- Apps.setEnvFromInputString(environment, conf.get(
+ TezYARNUtils.setEnvFromInputString(environment, conf.get(
MRJobConfig.MAPRED_ADMIN_USER_ENV,
- MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV));
+ MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV),
+ File.pathSeparator);
// Add the env variables passed by the user
String mapredChildEnv = (isMap ?
conf.get(MRJobConfig.MAP_ENV, "")
: conf.get(MRJobConfig.REDUCE_ENV, ""));
- Apps.setEnvFromInputString(environment, mapredChildEnv);
+ TezYARNUtils.setEnvFromInputString(environment, mapredChildEnv, File.pathSeparator);
// Set logging level in the environment.
environment.put(
@@ -869,12 +870,11 @@ public class MRHelpers {
* @param conf Configuration from which to extract information
* @param environment Environment map to update
*/
- public static void updateEnvironmentForMRAM(Configuration conf,
- Map<String, String> environment) {
- MRApps.setEnvFromInputString(environment,
- conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
- MRApps.setEnvFromInputString(environment,
- conf.get(MRJobConfig.MR_AM_ENV));
+ public static void updateEnvironmentForMRAM(Configuration conf, Map<String, String> environment) {
+ TezYARNUtils.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV),
+ File.pathSeparator);
+ TezYARNUtils.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ENV),
+ File.pathSeparator);
}
/**