You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by jl...@apache.org on 2016/10/13 22:22:50 UTC
tez git commit: TEZ-3466. Tez classpath building to mimic mapreduce
classpath building (jlowe)
Repository: tez
Updated Branches:
refs/heads/master b4be3619a -> 43f7b5e3a
TEZ-3466. Tez classpath building to mimic mapreduce classpath building (jlowe)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/43f7b5e3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/43f7b5e3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/43f7b5e3
Branch: refs/heads/master
Commit: 43f7b5e3a6b85e3e4fbb86c3bed81e5b586c79dd
Parents: b4be361
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Oct 13 22:22:07 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Oct 13 22:22:07 2016 +0000
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/common/TezYARNUtils.java | 36 +++++++++++++-------
.../apache/tez/dag/api/TezConfiguration.java | 15 ++++++--
.../org/apache/tez/common/TestTezYARNUtils.java | 13 +++++++
.../tez/mapreduce/hadoop/DeprecatedKeys.java | 4 +++
.../mapreduce/hadoop/TestDeprecatedKeys.java | 6 ++++
6 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 79b0e38..ef6b890 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3466. Tez classpath building to mimic mapreduce classpath building.
TEZ-3453. Correct the downloaded ATS dag data location for analyzer.
TEZ-3449. Fix Spelling typos.
TEZ-3464. Fix findbugs warnings in tez-dag mainLoop
http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index c505ca8..145f928 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -43,20 +43,13 @@ public class TezYARNUtils {
public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
StringBuilder classpathBuilder = new StringBuilder();
-
- // Add any additional user-specified classpath
- String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
- if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
- classpathBuilder.append(additionalClasspath)
- .append(File.pathSeparator);
+ boolean userClassesTakesPrecedence =
+ conf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST,
+ TezConfiguration.TEZ_USER_CLASSPATH_FIRST_DEFAULT);
+ if (userClassesTakesPrecedence) {
+ addUserSpecifiedClasspath(classpathBuilder, conf);
}
- // Add PWD:PWD/*
- classpathBuilder.append(Environment.PWD.$())
- .append(File.pathSeparator)
- .append(Environment.PWD.$() + File.separator + "*")
- .append(File.pathSeparator);
-
String [] tezLibUrisClassPath = conf.getStrings(TezConfiguration.TEZ_LIB_URIS_CLASSPATH);
if(!conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false) &&
@@ -107,10 +100,29 @@ public class TezYARNUtils {
.append(File.pathSeparator);
}
+ if (!userClassesTakesPrecedence) {
+ addUserSpecifiedClasspath(classpathBuilder, conf);
+ }
String classpath = classpathBuilder.toString();
return StringInterner.weakIntern(classpath);
}
+ private static void addUserSpecifiedClasspath(StringBuilder classpathBuilder,
+ Configuration conf) {
+ // Add any additional user-specified classpath
+ String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX);
+ if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) {
+ classpathBuilder.append(additionalClasspath)
+ .append(File.pathSeparator);
+ }
+
+ // Add PWD:PWD/*
+ classpathBuilder.append(Environment.PWD.$())
+ .append(File.pathSeparator)
+ .append(Environment.PWD.$() + File.separator + "*")
+ .append(File.pathSeparator);
+ }
+
public static void appendToEnvFromInputString(Map<String, String> env,
String envString, String classPathSeparator) {
if (envString != null && envString.length() > 0) {
http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 5aa8f7e..77ea4ff 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1094,11 +1094,22 @@ public class TezConfiguration extends Configuration {
public static final boolean TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT = false;
/**
+ * Boolean value.
+ * Specify whether the user classpath takes precedence over the Tez framework
+ * classpath.
+ */
+ @ConfigurationScope(Scope.CLIENT)
+ @ConfigurationProperty(type="boolean")
+ public static final String TEZ_USER_CLASSPATH_FIRST = TEZ_PREFIX + "user.classpath.first";
+ public static final boolean TEZ_USER_CLASSPATH_FIRST_DEFAULT = true;
+
+ /**
* String value.
*
* Specify additional classpath information to be used for Tez AM and all containers.
- * This will be prepended to the classpath before all framework specific components have been
- * specified.
+ * If {@link #TEZ_USER_CLASSPATH_FIRST} is true then this will be added to the classpath
+ * before all framework specific components have been specified, otherwise this will
+ * be added after the framework specific components.
*/
@ConfigurationScope(Scope.AM)
@ConfigurationProperty
http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
index 2dabf51..092bd50 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -43,6 +43,19 @@ public class TestTezYARNUtils {
classpath.indexOf(Environment.PWD.$()));
}
+ @Test(timeout = 20000)
+ public void testUserClasspathFirstFalse() {
+ Configuration conf = new Configuration(false);
+ conf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false);
+ conf.set(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX, "foobar");
+ String classpath = TezYARNUtils.getFrameworkClasspath(conf, true);
+ Assert.assertTrue(classpath.contains("foobar"));
+ Assert.assertTrue(classpath.indexOf("foobar") >
+ classpath.indexOf(TezConstants.TEZ_TAR_LR_NAME));
+ Assert.assertTrue(classpath.indexOf("foobar") >
+ classpath.indexOf(Environment.PWD.$()));
+ }
+
@Test(timeout = 5000)
public void testBasicArchiveClasspath() {
Configuration conf = new Configuration(false);
http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index a826f6d..9ae58c0 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -86,6 +86,8 @@ public class DeprecatedKeys {
TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS);
mrParamToDAGParamMap.put(MRJobConfig.JOB_TAGS,
TezConfiguration.TEZ_APPLICATION_TAGS);
+ mrParamToDAGParamMap.put(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST,
+ TezConfiguration.TEZ_USER_CLASSPATH_FIRST);
}
// TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild.
@@ -157,6 +159,8 @@ public class DeprecatedKeys {
registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS);
registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
+
+ registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, TezConfiguration.TEZ_USER_CLASSPATH_FIRST);
}
private static void addDeprecatedKeys() {
http://git-wip-us.apache.org/repos/asf/tez/blob/43f7b5e3/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index ae218a0..c574e38 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -43,6 +43,7 @@ public class TestDeprecatedKeys {
jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f);
jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true);
jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f);
+ jobConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
MRHelpers.translateMRConfToTez(jobConf);
@@ -64,6 +65,7 @@ public class TestDeprecatedKeys {
assertEquals(0.33f,
jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0),
0.01f);
+ assertEquals(false, jobConf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, true));
}
@Test(timeout = 5000)
@@ -76,6 +78,7 @@ public class TestDeprecatedKeys {
jobConf.setInt(MRJobConfig.IO_SORT_MB, 100);
jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100);
jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f);
+ jobConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000);
jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200);
@@ -100,6 +103,7 @@ public class TestDeprecatedKeys {
jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter");
jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator");
jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator");
+ jobConf.setBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, false);
jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true);
@@ -131,6 +135,7 @@ public class TestDeprecatedKeys {
assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false));
assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0.0f), 0.0f);
+ assertEquals(false, jobConf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, true));
assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD));
assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES));
@@ -156,6 +161,7 @@ public class TestDeprecatedKeys {
assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS));
assertNull(jobConf.get("map.sort.class"));
assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART));
+ assertNull(jobConf.get(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST));
}
}