You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by zh...@apache.org on 2017/03/28 19:48:24 UTC
[43/50] [abbrv] tez git commit: TEZ-3659. AM/Task classpath should
not contain hadoop conf directory. (harishjp)
TEZ-3659. AM/Task classpath should not contain hadoop conf directory. (harishjp)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/293c5933
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/293c5933
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/293c5933
Branch: refs/heads/TEZ-1190
Commit: 293c5933f27be96091320f4a5deb80815db33cfd
Parents: a339d05
Author: Harish JP <ha...@gmail.com>
Authored: Fri Mar 17 08:06:41 2017 +0530
Committer: Harish JP <ha...@gmail.com>
Committed: Fri Mar 17 08:07:55 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/tez/common/TezYARNUtils.java | 7 +++----
.../org/apache/tez/dag/api/TezConfiguration.java | 13 +++++++++++++
.../java/org/apache/tez/common/TestTezYARNUtils.java | 15 +++++++++++++++
3 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index d5da672..bd6de11 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -18,7 +18,6 @@
package org.apache.tez.common;
import java.io.File;
-import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -103,10 +102,10 @@ public class TezYARNUtils {
classpathBuilder.append(c.trim())
.append(File.pathSeparator);
}
- } else {
+ } else if (conf.getBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF,
+ TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT)) {
// Setup HADOOP_CONF_DIR after PWD and tez-libs, if it's required.
- classpathBuilder.append(Environment.HADOOP_CONF_DIR.$())
- .append(File.pathSeparator);
+ classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()).append(File.pathSeparator);
}
if (!userClassesTakesPrecedence) {
http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 5900f5e..40f84e6 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1158,6 +1158,19 @@ public class TezConfiguration extends Configuration {
TEZ_PREFIX + "cluster.additional.classpath.prefix";
/**
+ * Boolean value.
+ * If this value is true then tez explicitly adds hadoop conf directory into classpath for AM and
+ * task containers. Default is false.
+ */
+ @Private
+ @Unstable
+ @ConfigurationScope(Scope.CLIENT)
+ @ConfigurationProperty(type="boolean")
+ public static final String TEZ_CLASSPATH_ADD_HADOOP_CONF = TEZ_PREFIX +
+ "classpath.add-hadoop-conf";
+ public static final boolean TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT = false;
+
+ /**
* Session-related properties
*/
@Private
http://git-wip-us.apache.org/repos/asf/tez/blob/293c5933/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
index 9f02801..4fdd61b 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -70,6 +70,21 @@ public class TestTezYARNUtils {
Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*"));
Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
+ "lib" + File.separator + "*"));
+ Assert.assertTrue(!classpath.contains(Environment.HADOOP_CONF_DIR.$()));
+ Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
+ classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
+ }
+
+ @Test(timeout = 5000)
+ public void testNoHadoopConfInClasspath() {
+ Configuration conf = new Configuration(false);
+ conf.setBoolean(TezConfiguration.TEZ_CLASSPATH_ADD_HADOOP_CONF, true);
+ String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
+ Assert.assertTrue(classpath.contains(Environment.PWD.$()));
+ Assert.assertTrue(classpath.contains(Environment.PWD.$() + File.separator + "*"));
+ Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator + "*"));
+ Assert.assertTrue(classpath.contains(TezConstants.TEZ_TAR_LR_NAME + File.separator
+ + "lib" + File.separator + "*"));
Assert.assertTrue(classpath.contains(Environment.HADOOP_CONF_DIR.$()));
Assert.assertTrue(classpath.indexOf(Environment.PWD.$()) <
classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));