You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/04/30 04:19:12 UTC
git commit: TEZ-1085. Leave env values unchanged if they aren't set
on the client. Contributed by Rohini Palaniswamy.
Repository: incubator-tez
Updated Branches:
refs/heads/master a4344673a -> 9f04ae4d7
TEZ-1085. Leave env values unchanged if they aren't set on the client.
Contributed by Rohini Palaniswamy.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/9f04ae4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/9f04ae4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/9f04ae4d
Branch: refs/heads/master
Commit: 9f04ae4d7cb54e21fe43e1a11357eccdd1dad660
Parents: a434467
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Apr 29 19:17:37 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Apr 29 19:17:37 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/tez/common/TezYARNUtils.java | 13 +++++++------
.../org/apache/tez/mapreduce/hadoop/TestMRHelpers.java | 9 ++++-----
2 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f04ae4d/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index aceec81..bcd0393 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -27,15 +27,16 @@ import org.apache.hadoop.util.StringInterner;
@Private
public class TezYARNUtils {
+
+ private static Pattern ENV_VARIABLE_PATTERN = Pattern.compile(Shell.getEnvironmentVariableRegex());
public static void setEnvFromInputString(Map<String, String> env,
String envString, String classPathSeparator) {
if (envString != null && envString.length() > 0) {
String childEnvs[] = envString.split(",");
- Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex());
for (String cEnv : childEnvs) {
String[] parts = cEnv.split("="); // split on '='
- Matcher m = p.matcher(parts[1]);
+ Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]);
StringBuffer sb = new StringBuffer();
while (m.find()) {
String var = m.group(1);
@@ -45,10 +46,10 @@ public class TezYARNUtils {
// from the tt's env
if (replace == null)
replace = System.getenv(var);
- // the env key is note present anywhere .. simply set it
- if (replace == null)
- replace = "";
- m.appendReplacement(sb, Matcher.quoteReplacement(replace));
+ // If the env key is note present leave it as it is and assume it will
+ // be set by YARN ContainerLauncher. For eg: $HADOOP_COMMON_HOME
+ if (replace != null)
+ m.appendReplacement(sb, Matcher.quoteReplacement(replace));
}
m.appendTail(sb);
addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/9f04ae4d/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
index 7294ddc..cffded8 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
@@ -42,17 +42,14 @@ import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
+import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
-import org.apache.zookeeper.Environment.Entry;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -339,6 +336,8 @@ public class TestMRHelpers {
Assert.assertTrue(env.containsKey(Environment.LD_LIBRARY_PATH.name()));
Assert.assertTrue(env.containsKey(Environment.SHELL.name()));
Assert.assertTrue(env.containsKey("HADOOP_ROOT_LOGGER"));
+ Assert.assertEquals("$PWD:$HADOOP_COMMON_HOME/lib/native",
+ env.get(Environment.LD_LIBRARY_PATH.name()));
// TEZ-273 will reinstate this or similar.
// for (String val : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {