You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/12/22 21:30:59 UTC
tez git commit: TEZ-1878. Task-specific log level override not
working in certain conditions. (sseth)
Repository: tez
Updated Branches:
refs/heads/master 84315d559 -> a139989c0
TEZ-1878. Task-specific log level override not working in certain
conditions. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a139989c
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a139989c
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a139989c
Branch: refs/heads/master
Commit: a139989c0b43074514bf936511be4e5111a30344
Parents: 84315d5
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Dec 22 12:30:56 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Dec 22 12:30:56 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 7 +--
.../tez/dag/app/dag/impl/TestVertexImpl2.java | 66 ++++++++++++++++++++
3 files changed, 70 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a139989c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 804df1d..c010b57 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -14,6 +14,7 @@ Release 0.6.0: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-1878. Task-specific log level override not working in certain conditions.
TEZ-1886. remove deprecation warnings for tez-ui on the console.
TEZ-1875. dropdown filters do not work on vertices and task attempts page.
TEZ-1873. TestTezAMRMClient fails due to host resolution timing out.
http://git-wip-us.apache.org/repos/asf/tez/blob/a139989c/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 13993cc..b1c93da 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -779,12 +779,11 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
String [] taskLogParamsTaskSpecific = taskSpecificLaunchCmdOption.getTaskSpecificLogParams();
this.javaOptsTaskSpecific = TezClientUtils
.maybeAddDefaultLoggingJavaOpts(taskLogParamsTaskSpecific[0], javaOptsWithoutLoggerMods);
+
+ environmentTaskSpecific = new HashMap<String, String>(this.environment.size());
+ environmentTaskSpecific.putAll(environment);
if (taskLogParamsTaskSpecific.length == 2 && !Strings.isNullOrEmpty(taskLogParamsTaskSpecific[1])) {
- environmentTaskSpecific = new HashMap<String, String>(this.environment.size());
- environmentTaskSpecific.putAll(environment);
TezClientUtils.addLogParamsToEnv(environmentTaskSpecific, taskLogParamsTaskSpecific);
- } else {
- environmentTaskSpecific = null;
}
} else {
this.javaOptsTaskSpecific = null;
http://git-wip-us.apache.org/repos/asf/tez/blob/a139989c/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
index 80fa86a..b4064a0 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
@@ -183,6 +183,72 @@ public class TestVertexImpl2 {
}
}
+ @Test(timeout = 5000)
+ public void testTaskSpecificLoggingOpts2() {
+
+ String vertexName = "testvertex";
+ String customJavaOpts = "-Xmx128m";
+
+ Configuration conf = new TezConfiguration();
+ conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "WARN;org.apache.tez=INFO");
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG");
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);
+
+ LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);
+
+ // Expected command opts for regular tasks
+ List<String> expectedCommands = new LinkedList<String>();
+ expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+ expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
+ TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+ for (int i = 3 ; i < testInfo.numTasks ; i++) {
+ ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+ String javaOpts = containerContext.getJavaOpts();
+
+ assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+ for (String expectedCmd : expectedCommands) {
+ assertTrue(javaOpts.contains(expectedCmd));
+ }
+
+ Map<String, String> env = containerContext.getEnvironment();
+ String val = env.get(testInfo.envKey);
+ assertEquals(testInfo.envVal, val);
+ String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ assertEquals("org.apache.tez=INFO", logEnvVal);
+ }
+
+ // Expected command opts for instrumented tasks.
+ expectedCommands = new LinkedList<String>();
+ expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+ expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
+ TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+ for (int i = 0 ; i < 3 ; i++) {
+ ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+ String javaOpts = containerContext.getJavaOpts();
+
+ assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+ for (String expectedCmd : expectedCommands) {
+ assertTrue(javaOpts.contains(expectedCmd));
+ }
+
+ Map<String, String> env = containerContext.getEnvironment();
+ String val = env.get(testInfo.envKey);
+ assertEquals(testInfo.envVal, val);
+ String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ assertNull(logEnvVal);
+ }
+ }
+
+
private static class LogTestInfoHolder {
final AppContext mockAppContext;