You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2015/10/29 17:22:56 UTC
tez git commit: TEZ-2679. Admin forms of launch env settings (jeagles)
Repository: tez
Updated Branches:
refs/heads/master 65f457500 -> 19808b7cd
TEZ-2679. Admin forms of launch env settings (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/19808b7c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/19808b7c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/19808b7c
Branch: refs/heads/master
Commit: 19808b7cd80f267a3474102cd0f761155a6df239
Parents: 65f4575
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Thu Oct 29 11:22:42 2015 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Thu Oct 29 11:22:42 2015 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../org/apache/tez/client/TezClientUtils.java | 10 ++++--
.../org/apache/tez/common/TezYARNUtils.java | 20 ++++++------
.../main/java/org/apache/tez/dag/api/DAG.java | 5 ++-
.../apache/tez/dag/api/TezConfiguration.java | 34 ++++++++++++++++++--
.../org/apache/tez/common/TestTezYARNUtils.java | 23 +++++++++++++
6 files changed, 78 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/19808b7c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b08071d..809ebf9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
Release 0.8.2: Unreleased
INCOMPATIBLE CHANGES
+ TEZ-2679. Admin forms of launch env settings
ALL CHANGES:
TEZ-2906. Compilation fails with hadoop 2.2.0
@@ -225,6 +226,7 @@ TEZ-2003: Support for External services CHANGES
Release 0.7.1: Unreleased
INCOMPATIBLE CHANGES
+ TEZ-2679. Admin forms of launch env settings
ALL CHANGES
TEZ-2900. Ignore V_INPUT_DATA_INFORMATION when vertex is in Failed/Killed/Error
http://git-wip-us.apache.org/repos/asf/tez/blob/19808b7c/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index c96a7ec..43a97fa 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -508,9 +508,13 @@ public class TezClientUtils {
}
Map<String, String> environment = new TreeMap<String, String>();
- TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV,
- TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, tezLrsAsArchive);
-
+ TezYARNUtils.setupDefaultEnv(environment, conf,
+ TezConfiguration.TEZ_AM_LAUNCH_ENV,
+ TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT,
+ TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV,
+ TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT,
+ tezLrsAsArchive);
+
addVersionInfoToEnv(environment, apiVersionInfo);
addLogParamsToEnv(environment, amLogParams);
http://git-wip-us.apache.org/repos/asf/tez/blob/19808b7c/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index fc30780..d7093db 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -167,8 +167,8 @@ public class TezYARNUtils {
}
}
- public static void setupDefaultEnv(Map<String, String> env,
- Configuration conf, String confEnvKey, String confEnvKeyDefault, boolean usingArchive) {
+ public static void setupDefaultEnv(Map<String, String> env, Configuration conf, String userEnvKey, String userEnvDefault,
+ String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) {
// Setup the CLASSPATH in environment
// i.e. add { Hadoop jars, job jar, CWD } to classpath.
String classpath = getFrameworkClasspath(conf, usingArchive);
@@ -176,14 +176,16 @@ public class TezYARNUtils {
ApplicationConstants.Environment.CLASSPATH.name(),
classpath, File.pathSeparator);
+ // Pre-pend pwd to LD_LIBRARY_PATH
+ // Done separately here because this is known to work platform
+ // independent
+ TezYARNUtils.addToEnvironment(env,
+ Environment.LD_LIBRARY_PATH.name(), Environment.PWD.$(), File.pathSeparator);
+ TezYARNUtils.appendToEnvFromInputString(env,
+ conf.get(userEnvKey, userEnvDefault), File.pathSeparator);
// set any env from config if it is not set already
- TezYARNUtils.setEnvIfAbsentFromInputString(env, conf.get(
- confEnvKey, confEnvKeyDefault));
-
- // Append pwd to LD_LIBRARY_PATH
- // Done separately here because this is known to work platform independent
- TezYARNUtils.addToEnvironment(env, Environment.LD_LIBRARY_PATH.name(),
- Environment.PWD.$(), File.pathSeparator);
+ TezYARNUtils.appendToEnvFromInputString(env,
+ conf.get(clusterDefaultEnvKey, clusterDefaultEnvDefault), File.pathSeparator);
}
public static void replaceInEnv(Map<String, String> env, String key, String value) {
http://git-wip-us.apache.org/repos/asf/tez/blob/19808b7c/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index c02d86f..310a795 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -910,7 +910,10 @@ public class DAG {
Map<String, String> taskEnv = Maps.newHashMap(vertex.getTaskEnvironment());
TezYARNUtils.setupDefaultEnv(taskEnv, tezConf,
TezConfiguration.TEZ_TASK_LAUNCH_ENV,
- TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT, tezLrsAsArchive);
+ TezConfiguration.TEZ_TASK_LAUNCH_ENV_DEFAULT,
+ TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_ENV,
+ TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT,
+ tezLrsAsArchive);
for (Map.Entry<String, String> entry : taskEnv.entrySet()) {
PlanKeyValuePair.Builder envSettingBuilder = PlanKeyValuePair.newBuilder();
envSettingBuilder.setKey(entry.getKey());
http://git-wip-us.apache.org/repos/asf/tez/blob/19808b7c/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 80b9860..ac3dd4a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -359,18 +359,45 @@ public class TezConfiguration extends Configuration {
"PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin":
"LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_COMMON_HOME/lib/native/";
+ /**
+ * String value. Env settings will be merged with {@link #TEZ_AM_LAUNCH_ENV}
+ * during the launch of the AppMaster process. This property will typically be configured to
+ * include default system env meant to be used by all jobs in a cluster. If required, the values can
+ * be appended to per job.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV =
+ TEZ_AM_PREFIX + "launch.cluster-default.env";
+ public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT =
+ NATIVE_LIB_PARAM_DEFAULT;
+
/** String value. Env settings for the Tez AppMaster process.
* Should be specified as a comma-separated of key-value pairs where each pair
* is defined as KEY=VAL
* e.g. "LD_LIBRARY_PATH=.,USERNAME=foo"
* These take least precedence compared to other methods of setting env.
* These get added to the app master environment prior to launching it.
- */
+ * This setting will prepend existing settings in the cluster default
+ */
@ConfigurationScope(Scope.AM)
@ConfigurationProperty
public static final String TEZ_AM_LAUNCH_ENV = TEZ_AM_PREFIX
+ "launch.env";
- public static final String TEZ_AM_LAUNCH_ENV_DEFAULT = NATIVE_LIB_PARAM_DEFAULT;
+ public static final String TEZ_AM_LAUNCH_ENV_DEFAULT = "";
+
+ /**
+ * String value. Env settings will be merged with {@link #TEZ_TASK_LAUNCH_ENV}
+ * during the launch of the task process. This property will typically be configured to
+ * include default system env meant to be used by all jobs in a cluster. If required, the values can
+ * be appended to per job.
+ */
+ @ConfigurationScope(Scope.VERTEX)
+ @ConfigurationProperty
+ public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_ENV =
+ TEZ_TASK_PREFIX + "launch.cluster-default.env";
+ public static final String TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT =
+ NATIVE_LIB_PARAM_DEFAULT;
/** String value. Env settings for the Tez Task processes.
* Should be specified as a comma-separated of key-value pairs where each pair
@@ -378,12 +405,13 @@ public class TezConfiguration extends Configuration {
* e.g. "LD_LIBRARY_PATH=.,USERNAME=foo"
* These take least precedence compared to other methods of setting env
* These get added to the task environment prior to launching it.
+ * This setting will prepend existing settings in the cluster default
*/
@ConfigurationScope(Scope.VERTEX)
@ConfigurationProperty
public static final String TEZ_TASK_LAUNCH_ENV = TEZ_TASK_PREFIX
+ "launch.env";
- public static final String TEZ_TASK_LAUNCH_ENV_DEFAULT = NATIVE_LIB_PARAM_DEFAULT;
+ public static final String TEZ_TASK_LAUNCH_ENV_DEFAULT = "";
@Private
@ConfigurationScope(Scope.DAG)
http://git-wip-us.apache.org/repos/asf/tez/blob/19808b7c/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
index 6534cb6..6e9e06c 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -19,6 +19,8 @@
package org.apache.tez.common;
import java.io.File;
+import java.util.Map;
+import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -57,4 +59,25 @@ public class TestTezYARNUtils {
classpath.indexOf(Environment.HADOOP_CONF_DIR.$()));
}
+ @Test(timeout = 5000)
+ public void testSetupDefaultEnvironment() {
+ Configuration conf = new Configuration(false);
+ conf.set(TezConfiguration.TEZ_AM_LAUNCH_ENV, "LD_LIBRARY_PATH=USER_PATH,USER_KEY=USER_VALUE");
+ conf.set(TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV, "LD_LIBRARY_PATH=DEFAULT_PATH,DEFAULT_KEY=DEFAULT_VALUE");
+
+ Map<String, String> environment = new TreeMap<String, String>();
+ TezYARNUtils.setupDefaultEnv(environment, conf,
+ TezConfiguration.TEZ_AM_LAUNCH_ENV,
+ TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT,
+ TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV,
+ TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_ENV_DEFAULT, false);
+
+ String value1 = environment.get("USER_KEY");
+ Assert.assertEquals("User env should merge with default env", "USER_VALUE", value1);
+ String value2 = environment.get("DEFAULT_KEY");
+ Assert.assertEquals("User env should merge with default env", "DEFAULT_VALUE", value2);
+ String value3 = environment.get("LD_LIBRARY_PATH");
+ Assert.assertEquals("User env should append default env",
+ Environment.PWD.$() + File.pathSeparator + "USER_PATH" + File.pathSeparator + "DEFAULT_PATH", value3);
+ }
}