You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/12/22 22:00:27 UTC
tez git commit: TEZ-1775. Allow setting log level per logger. (sseth)
Repository: tez
Updated Branches:
refs/heads/branch-0.5 5972c0623 -> 599bb0fe2
TEZ-1775. Allow setting log level per logger. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/599bb0fe
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/599bb0fe
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/599bb0fe
Branch: refs/heads/branch-0.5
Commit: 599bb0fe2ee6d80125c10eecc2554ddea8f869ef
Parents: 5972c06
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Dec 22 13:00:21 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Dec 22 13:00:21 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/client/TezClientUtils.java | 47 +++-
.../apache/tez/dag/api/TezConfiguration.java | 53 ++++-
.../org/apache/tez/dag/api/TezConstants.java | 2 +
.../apache/tez/client/TestTezClientUtils.java | 82 ++++++-
.../apache/tez/common/TezLog4jConfigurator.java | 50 ++++
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 50 +++-
.../dag/utils/TaskSpecificLaunchCmdOption.java | 94 +++++---
.../resources/tez-container-log4j.properties | 2 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 2 +-
.../tez/dag/app/dag/impl/TestVertexImpl2.java | 231 +++++++++++++++++++
.../utils/TestTaskSpecificLaunchCmdOption.java | 46 +++-
12 files changed, 607 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d0abe99..8dc8aaa 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
Release 0.5.4: Unreleased
ALL CHANGES:
+ TEZ-1775. Allow setting log level per logger.
TEZ-1851. FileSystem counters do not differentiate between different FileSystems.
TEZ-1852. Get examples to work in LocalMode.
TEZ-1861. Fix failing test: TestOnFileSortedOutput.
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index de4bdd0..4cac4d2 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -36,6 +36,7 @@ import java.util.TreeMap;
import java.util.Vector;
import java.util.Map.Entry;
+import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -402,7 +403,9 @@ public class TezClientUtils {
* @throws IOException
* @throws YarnException
*/
- static ApplicationSubmissionContext createApplicationSubmissionContext(
+ @Private
+ @VisibleForTesting
+ public static ApplicationSubmissionContext createApplicationSubmissionContext(
ApplicationId appId, DAG dag, String amName,
AMConfiguration amConfig, Map<String, LocalResource> tezJarResources,
Credentials sessionCreds, boolean tezLrsAsArchive,
@@ -440,7 +443,7 @@ public class TezClientUtils {
}
// Add Staging dir creds to the list of session credentials.
- TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] {binaryConfPath}, conf);
+ TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[]{binaryConfPath}, conf);
// Add session specific credentials to the AM credentials.
amLaunchCredentials.mergeAll(sessionCreds);
@@ -456,11 +459,15 @@ public class TezClientUtils {
String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability);
vargs.add(amOpts);
- String amLogLevel = amConfig.getTezConfiguration().get(
+ String amLogLevelString = amConfig.getTezConfiguration().get(
TezConfiguration.TEZ_AM_LOG_LEVEL,
TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT);
+ String[] amLogParams = parseLogParams(amLogLevelString);
+
+ String amLogLevel = amLogParams[0];
maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs);
+
// FIX sun bug mentioned in TEZ-327
vargs.add("-Dsun.nio.ch.bugLevel=''");
@@ -502,7 +509,8 @@ public class TezClientUtils {
}
addVersionInfoToEnv(environment, apiVersionInfo);
-
+ addLogParamsToEnv(environment, amLogParams);
+
Map<String, LocalResource> amLocalResources =
new TreeMap<String, LocalResource>();
@@ -650,8 +658,9 @@ public class TezClientUtils {
}
TezClientUtils.addLog4jSystemProperties(logLevel, vargs);
}
-
- static String maybeAddDefaultLoggingJavaOpts(String logLevel, String javaOpts) {
+
+ @Private
+ public static String maybeAddDefaultLoggingJavaOpts(String logLevel, String javaOpts) {
List<String> vargs = new ArrayList<String>(5);
if (javaOpts != null) {
vargs.add(javaOpts);
@@ -679,10 +688,6 @@ public class TezClientUtils {
// Add options specified in the DAG at the end.
vOpts = vConfigOpts + " " + vOpts;
}
-
- vOpts = maybeAddDefaultLoggingJavaOpts(conf.get(
- TezConfiguration.TEZ_TASK_LOG_LEVEL,
- TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT), vOpts);
return vOpts;
}
@@ -690,6 +695,7 @@ public class TezClientUtils {
@VisibleForTesting
public static void addLog4jSystemProperties(String logLevel,
List<String> vargs) {
+ vargs.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
vargs.add("-Dlog4j.configuration="
+ TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
@@ -920,4 +926,25 @@ public class TezClientUtils {
}
}
+ @Private
+ public static void addLogParamsToEnv(Map<String, String> environment, String[] logParams) {
+ if (logParams.length == 2 && !Strings.isNullOrEmpty(logParams[1])) {
+ TezYARNUtils.replaceInEnv(environment, TezConstants.TEZ_CONTAINER_LOG_PARAMS, logParams[1]);
+ }
+ }
+
+ @Private
+ public static String[] parseLogParams(String logConfig) {
+ if (!Strings.isNullOrEmpty(logConfig)) {
+ int separatorIndex = logConfig.indexOf(TezConstants.TEZ_CONTAINER_LOG_PARAMS_SEPARATOR);
+ if (separatorIndex == -1) {
+ return new String[]{logConfig.trim()};
+ } else {
+ return new String[]{logConfig.substring(0, separatorIndex),
+ logConfig.substring(separatorIndex + 1, logConfig.length()).trim()};
+ }
+ } else {
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index fd92b1f..e4ca689 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -117,11 +117,41 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_SESSION_MODE = TEZ_AM_PREFIX + "mode.session";
public static boolean TEZ_AM_SESSION_MODE_DEFAULT = false;
- /** Root Logging level passed to the Tez app master.*/
+ /**
+ * Root Logging level passed to the Tez app master.
+ *
+ * Simple configuration: Set the log level for all loggers.
+ * e.g. INFO
+ * This sets the log level to INFO for all loggers.
+ *
+ * Advanced configuration: Set the log level for all classes, along with a different level for some.
+ * e.g. DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO
+ * This sets the log level for all loggers to DEBUG, expect for the
+ * org.apache.hadoop.ipc and org.apache.hadoop.security, which are set to INFO
+ *
+ * Note: The global log level must always be the first parameter.
+ * DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is valid
+ * org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid
+ * */
public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX + "log.level";
public static final String TEZ_AM_LOG_LEVEL_DEFAULT = "INFO";
- /** Root Logging level passed to the Tez tasks.*/
+ /**
+ * Root Logging level passed to the Tez tasks.
+ *
+ * Simple configuration: Set the log level for all loggers.
+ * e.g. INFO
+ * This sets the log level to INFO for all loggers.
+ *
+ * Advanced configuration: Set the log level for all classes, along with a different level for some.
+ * e.g. DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO
+ * This sets the log level for all loggers to DEBUG, expect for the
+ * org.apache.hadoop.ipc and org.apache.hadoop.security, which are set to INFO
+ *
+ * Note: The global log level must always be the first parameter.
+ * DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is valid
+ * org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid
+ * */
public static final String TEZ_TASK_LOG_LEVEL = TEZ_TASK_PREFIX + "log.level";
public static final String TEZ_TASK_LOG_LEVEL_DEFAULT = "INFO";
@@ -726,6 +756,25 @@ public class TezConfiguration extends Configuration {
".launch.cmd-opts";
/**
+ * Task specific log level.
+ *
+ * Simple configuration: Set the log level for all loggers.
+ * e.g. INFO
+ * This sets the log level to INFO for all loggers.
+ *
+ * Advanced configuration: Set the log level for all classes, along with a different level for some.
+ * e.g. DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO
+ * This sets the log level for all loggers to DEBUG, expect for the
+ * org.apache.hadoop.ipc and org.apache.hadoop.security, which are set to INFO
+ *
+ * Note: The global log level must always be the first parameter.
+ * DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is valid
+ * org.apache.hadoop.ipc=INFO;org.apache.hadoop.security=INFO is not valid
+ * */
+ @Unstable
+ public static final String TEZ_TASK_SPECIFIC_LOG_LEVEL = TEZ_PREFIX + "task-specific" + ".log.level";
+
+ /**
* String value that is a class name.
* Specify the class to use for logging history data
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index e335da8..bc4208f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -62,6 +62,8 @@ public class TezConstants {
public static final String TEZ_CONTAINER_LOG_FILE_NAME = "syslog";
public static final String TEZ_CONTAINER_ERR_FILE_NAME = "stderr";
public static final String TEZ_CONTAINER_OUT_FILE_NAME = "stdout";
+ public static final String TEZ_CONTAINER_LOG_PARAMS = "TEZ_CONTAINER_LOG_PARAMS";
+ public static final String TEZ_CONTAINER_LOG_PARAMS_SEPARATOR = ";";
public static final String TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME =
TezConfiguration.TEZ_SESSION_PREFIX + "local-resources.pb";
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index f39c4ea..c8e144f 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -20,6 +20,7 @@ package org.apache.tez.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -29,6 +30,7 @@ import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -41,12 +43,20 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.records.DAGProtos;
import org.junit.Assert;
import org.junit.Test;
@@ -183,6 +193,75 @@ public class TestTezClientUtils {
assertFalse(localizedMap.isEmpty());
}
+ @Test(timeout = 5000)
+ public void testAMLoggingOptsSimple() throws IOException, YarnException {
+
+ TezConfiguration tezConf = new TezConfiguration();
+ tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN");
+
+ ApplicationId appId = ApplicationId.newInstance(1000, 1);
+ DAG dag = DAG.create("testdag");
+ dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"))
+ .setTaskLaunchCmdOpts("initialLaunchOpts"));
+ AMConfiguration amConf =
+ new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), new Credentials());
+ ApplicationSubmissionContext appSubmissionContext =
+ TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
+ new HashMap<String, LocalResource>(), new Credentials(), false, new TezApiVersionInfo());
+
+ List<String> expectedCommands = new LinkedList<String>();
+ expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+ expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
+ TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+ List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
+ assertEquals(1, commands.size());
+ for (String expectedCmd : expectedCommands) {
+ assertTrue(commands.get(0).contains(expectedCmd));
+ }
+
+ Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
+ String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ assertNull(logEnv);
+ }
+
+ @Test(timeout = 5000)
+ public void testAMLoggingOptsPerLogger() throws IOException, YarnException {
+
+ TezConfiguration tezConf = new TezConfiguration();
+ tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
+
+ ApplicationId appId = ApplicationId.newInstance(1000, 1);
+ DAG dag = DAG.create("testdag");
+ dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"))
+ .setTaskLaunchCmdOpts("initialLaunchOpts"));
+ AMConfiguration amConf =
+ new AMConfiguration(tezConf, new HashMap<String, LocalResource>(), new Credentials());
+ ApplicationSubmissionContext appSubmissionContext =
+ TezClientUtils.createApplicationSubmissionContext(appId, dag, "amName", amConf,
+ new HashMap<String, LocalResource>(), new Credentials(), false, new TezApiVersionInfo());
+
+ List<String> expectedCommands = new LinkedList<String>();
+ expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+ expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
+ TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+ List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
+ assertEquals(1, commands.size());
+ for (String expectedCmd : expectedCommands) {
+ assertTrue(commands.get(0).contains(expectedCmd));
+ }
+
+ Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
+ String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ assertEquals("org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG", logEnv);
+ }
@Test(timeout = 5000)
public void testAMCommandOpts() {
@@ -327,7 +406,8 @@ public class TestTezClientUtils {
String javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts("FOOBAR", origJavaOpts);
Assert.assertNotNull(javaOpts);
Assert.assertTrue(javaOpts.contains("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=FOOBAR")
- && javaOpts.contains(TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE));
+ && javaOpts.contains(TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE)
+ && javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"));
}
// To run this test case see TestTezCommonUtils::testLocalResourceVisibility
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java b/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java
new file mode 100644
index 0000000..24731c2
--- /dev/null
+++ b/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common;
+
+import java.util.Properties;
+
+
+import org.apache.log4j.PropertyConfigurator;
+import org.apache.log4j.spi.LoggerRepository;
+import org.apache.tez.dag.api.TezConstants;
+
+public class TezLog4jConfigurator extends PropertyConfigurator {
+
+ public void doConfigure(Properties properties, LoggerRepository repository) {
+ String logParams = System.getenv(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ if (logParams != null) {
+ String []parts = logParams.split(TezConstants.TEZ_CONTAINER_LOG_PARAMS_SEPARATOR);
+ for (String logParam : parts) {
+ String [] logParamParts = logParam.split("=");
+ if (logParamParts.length == 2) {
+ String loggerName = "log4j.logger." + logParamParts[0];
+ String logLevel = logParamParts[1].toUpperCase();
+ properties.setProperty(loggerName, logLevel);
+ } else {
+ // Cannot use Log4J logging from here.
+ System.out.println("TezLog4jConfigurator Ignoring invalid log parameter [" + logParam + "]");
+ continue;
+ }
+ }
+ }
+ super.doConfigure(properties, repository);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 593ecca..9137b1e 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -39,6 +39,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import javax.annotation.Nullable;
+import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
@@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.client.TezClientUtils;
import org.apache.tez.common.ATSConstants;
import org.apache.tez.common.ReflectionUtils;
import org.apache.tez.common.counters.TezCounters;
@@ -68,6 +70,7 @@ import org.apache.tez.dag.api.OutputCommitterDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
import org.apache.tez.dag.api.RootInputLeafOutput;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.TaskLocationHint;
@@ -691,7 +694,9 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private TaskLocationHint taskLocationHints[];
private Map<String, LocalResource> localResources;
- private Map<String, String> environment;
+ private final Map<String, String> environment;
+ private final Map<String, String> environmentTaskSpecific;
+ private final String javaOptsTaskSpecific;
private final String javaOpts;
private final ContainerContext containerContext;
private VertexTerminationCause terminationCause;
@@ -753,9 +758,36 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
this.environment = DagTypeConverters
.createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig()
.getEnvironmentSettingList());
- this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan
- .getTaskConfig().getJavaOpts() : null;
this.taskSpecificLaunchCmdOpts = taskSpecificLaunchCmdOption;
+
+ // Set up log properties, including task specific log properties.
+ String javaOptsWithoutLoggerMods =
+ vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan.getTaskConfig().getJavaOpts() : null;
+ String logString = conf.get(TezConfiguration.TEZ_TASK_LOG_LEVEL, TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT);
+ String [] taskLogParams = TezClientUtils.parseLogParams(logString);
+ this.javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts(taskLogParams[0], javaOptsWithoutLoggerMods);
+
+ if (taskSpecificLaunchCmdOpts.hasModifiedLogProperties()) {
+ String [] taskLogParamsTaskSpecific = taskSpecificLaunchCmdOption.getTaskSpecificLogParams();
+ this.javaOptsTaskSpecific = TezClientUtils
+ .maybeAddDefaultLoggingJavaOpts(taskLogParamsTaskSpecific[0], javaOptsWithoutLoggerMods);
+ if (taskLogParamsTaskSpecific.length == 2 && !Strings.isNullOrEmpty(taskLogParamsTaskSpecific[1])) {
+ environmentTaskSpecific = new HashMap<String, String>(this.environment.size());
+ environmentTaskSpecific.putAll(environment);
+ TezClientUtils.addLogParamsToEnv(environmentTaskSpecific, taskLogParamsTaskSpecific);
+ } else {
+ environmentTaskSpecific = null;
+ }
+ } else {
+ this.javaOptsTaskSpecific = null;
+ this.environmentTaskSpecific = null;
+ }
+
+ // env for tasks which don't have task-specific configuration. Has to be set up later to
+ // optionally allow copying this for specific tasks
+ TezClientUtils.addLogParamsToEnv(this.environment, taskLogParams);
+
+
this.containerContext = new ContainerContext(this.localResources,
appContext.getCurrentDAG().getCredentials(), this.environment, this.javaOpts, this);
@@ -1934,9 +1966,17 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@VisibleForTesting
ContainerContext getContainerContext(int taskIdx) {
if (taskSpecificLaunchCmdOpts.addTaskSpecificLaunchCmdOption(vertexName, taskIdx)) {
- String jvmOpts = taskSpecificLaunchCmdOpts.getTaskSpecificOption(javaOpts, vertexName, taskIdx);
+
+ String jvmOpts = javaOptsTaskSpecific != null ? javaOptsTaskSpecific : javaOpts;
+
+ if (taskSpecificLaunchCmdOpts.hasModifiedTaskLaunchOpts()) {
+ jvmOpts = taskSpecificLaunchCmdOpts.getTaskSpecificOption(jvmOpts, vertexName, taskIdx);
+ }
+
ContainerContext context = new ContainerContext(this.localResources,
- appContext.getCurrentDAG().getCredentials(), this.environment, jvmOpts);
+ appContext.getCurrentDAG().getCredentials(),
+ this.environmentTaskSpecific != null ? this.environmentTaskSpecific : this.environment,
+ jvmOpts);
return context;
} else {
return this.containerContext;
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
index a487a3b..59aede8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java
@@ -27,6 +27,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.TezClientUtils;
import org.apache.tez.dag.api.TezConfiguration;
import com.google.common.base.Splitter;
@@ -54,20 +55,22 @@ public class TaskSpecificLaunchCmdOption {
//Range regex where ':' should always be prepended and appended with digit.
final static Pattern RANGE_REGEX = Pattern.compile("(\\d+):(\\d+)");
- private Map<String, BitSet> tasksMap;
+ private final Map<String, BitSet> tasksMap;
//Task specific launch-cmd options
- private String tsLaunchCmdOpts;
+ private final String tsLaunchCmdOpts;
+ //Task specific log options
+ private final String[] tsLogParams;
public TaskSpecificLaunchCmdOption(Configuration conf) {
- tsLaunchCmdOpts =
- conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "");
+ this.tsLaunchCmdOpts =
+ conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS);
+ this.tsLogParams = TezClientUtils
+ .parseLogParams(conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL));
- if (!Strings.isNullOrEmpty(tsLaunchCmdOpts)) {
+ if (shouldParseSpecificTaskList()) {
this.tasksMap = getSpecificTasks(conf);
- if (!tasksMap.isEmpty() && tsLaunchCmdOpts.isEmpty()) {
- LOG.warn(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS
- + " should be specified for adding task specific launch-cmd options");
- }
+ } else {
+ this.tasksMap = null;
}
}
@@ -82,17 +85,40 @@ public class TaskSpecificLaunchCmdOption {
*/
public String getTaskSpecificOption(String launchCmdOpts, String vertexName,
int taskIdx) {
- launchCmdOpts = (launchCmdOpts == null) ? "" : launchCmdOpts;
- vertexName = vertexName.replaceAll(" ", "");
- String result =
- this.tsLaunchCmdOpts.replaceAll("__VERTEX_NAME__", vertexName)
- .replaceAll("__TASK_INDEX__", Integer.toString(taskIdx));
- result = (launchCmdOpts + " " + result);
+ if (this.tsLaunchCmdOpts != null) {
+ launchCmdOpts = (launchCmdOpts == null) ? "" : launchCmdOpts;
+ vertexName = vertexName.replaceAll(" ", "");
+ String result =
+ this.tsLaunchCmdOpts.replaceAll("__VERTEX_NAME__", vertexName)
+ .replaceAll("__TASK_INDEX__", Integer.toString(taskIdx));
+ result = (launchCmdOpts + " " + result);
+
+ LOG.info("Launch-cmd options added to vertexName=" + vertexName
+ + ", taskIdx=" + taskIdx + ", tsLaunchCmdOpts=" + result.trim());
+
+ return result.trim();
+ }
+ return launchCmdOpts;
+ }
- LOG.info("Launch-cmd options added to vertexName=" + vertexName
- + ", taskIdx=" + taskIdx + ", tsLaunchCmdOpts=" + result.trim());
+ public boolean hasModifiedTaskLaunchOpts() {
+ return !Strings.isNullOrEmpty(tsLaunchCmdOpts);
+ }
+
+ /**
+ * Retrieve a parsed form of the log string specified for per-task usage. </p>
+ * The first element of the array is the general log level. </p>
+ * The second level, if it exists, is the additional per logger configuration.
+ *
+ *
+ * @return parsed form of the log string specified. null if none specified
+ */
+ public String[] getTaskSpecificLogParams() {
+ return this.tsLogParams;
+ }
- return result.trim();
+ public boolean hasModifiedLogProperties() {
+ return this.tsLogParams != null;
}
/**
@@ -111,6 +137,10 @@ public class TaskSpecificLaunchCmdOption {
return (taskSet == null) ? false : ((taskSet.isEmpty()) ? true : taskSet.get(taskId));
}
+ private boolean shouldParseSpecificTaskList() {
+ return !(Strings.isNullOrEmpty(tsLaunchCmdOpts) && tsLogParams == null);
+ }
+
/**
* <pre>
* Get the set of tasks in the job, which needs task specific launch command arguments to be
@@ -123,23 +153,25 @@ public class TaskSpecificLaunchCmdOption {
* </pre>
*
* @param conf
- * @return Map<String, BitSet>
+ * @return a map from the vertex name to a BitSet representing tasks to be instruemented. null if
+ * the provided configuration is empty or invalid
*/
private Map<String, BitSet> getSpecificTasks(Configuration conf) {
String specificTaskList =
- conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "");
- final Map<String, BitSet> resultSet = new HashMap<String, BitSet>();
- if (specificTaskList.isEmpty() || !isValid(specificTaskList)) {
- return resultSet; // empty set
- }
- Matcher matcher = TASKS_REGEX.matcher(specificTaskList);
- while (matcher.find()) {
- String vertexName = matcher.group(1).trim();
- BitSet taskSet = parseTasks(matcher.group(2).trim());
- resultSet.put(vertexName, taskSet);
+ conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST);
+ if (!Strings.isNullOrEmpty(specificTaskList) && isValid(specificTaskList)) {
+ final Map<String, BitSet> resultSet = new HashMap<String, BitSet>();
+ Matcher matcher = TASKS_REGEX.matcher(specificTaskList);
+ while (matcher.find()) {
+ String vertexName = matcher.group(1).trim();
+ BitSet taskSet = parseTasks(matcher.group(2).trim());
+ resultSet.put(vertexName, taskSet);
+ }
+ LOG.info("Specific tasks with additional launch-cmd options=" + resultSet);
+ return resultSet;
+ } else {
+ return null;
}
- LOG.info("Specific tasks with additional launch-cmd options=" + resultSet);
- return resultSet;
}
private boolean isValid(String specificTaskList) {
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/main/resources/tez-container-log4j.properties
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/resources/tez-container-log4j.properties b/tez-dag/src/main/resources/tez-container-log4j.properties
index 8a11f2d..7a2aeab 100644
--- a/tez-dag/src/main/resources/tez-container-log4j.properties
+++ b/tez-dag/src/main/resources/tez-container-log4j.properties
@@ -12,7 +12,7 @@
# limitations under the License. See accompanying LICENSE file.
#
# Define some default values that can be overridden by system properties
-hadoop.root.logger=DEBUG,CLA
+tez.root.logger=INFO,CLA
# Define the root logger to the system property "tez.root.logger".
log4j.rootLogger=${tez.root.logger}, EventCounter
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 9500c97..3424e70 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -2271,7 +2271,7 @@ public class TestVertexImpl {
VertexImpl v3 = vertices.get("vertex3");
Assert.assertEquals("x3.y3", v3.getProcessorName());
- Assert.assertEquals("foo", v3.getJavaOpts());
+ Assert.assertTrue(v3.getJavaOpts().contains("foo"));
Assert.assertEquals(2, v3.getInputSpecList(0).size());
Assert.assertEquals(2, v3.getInputVerticesCount());
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
new file mode 100644
index 0000000..80fa86a
--- /dev/null
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.dag.api.TaskLocationHint;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.VertexLocationHint;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.AppContext;
+import org.apache.tez.dag.app.ContainerContext;
+import org.apache.tez.dag.app.TaskAttemptListener;
+import org.apache.tez.dag.app.TaskHeartbeatHandler;
+import org.apache.tez.dag.app.dag.DAG;
+import org.apache.tez.dag.app.dag.StateChangeNotifier;
+import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.dag.utils.TaskSpecificLaunchCmdOption;
+import org.junit.Test;
+
+/**
+ * Contains additional tests for VertexImpl. Please avoid adding class parameters.
+ */
+public class TestVertexImpl2 {
+
+ @Test(timeout = 5000)
+ public void testTaskLoggingOptsPerLogger() {
+
+ Configuration conf = new TezConfiguration();
+ conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO");
+
+ LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);
+
+ List<String> expectedCommands = new LinkedList<String>();
+ expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+ expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
+ TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+ for (int i = 0 ; i < testInfo.numTasks ; i++) {
+ ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+ String javaOpts = containerContext.getJavaOpts();
+ assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+ for (String expectedCmd : expectedCommands) {
+ assertTrue(javaOpts.contains(expectedCmd));
+ }
+
+ Map<String, String> env = containerContext.getEnvironment();
+ String val = env.get(testInfo.envKey);
+ assertEquals(testInfo.envVal, val);
+ String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ assertEquals("org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO", logEnvVal);
+ }
+ }
+
+ @Test(timeout = 5000)
+ public void testTaskLoggingOptsSimple() {
+
+ Configuration conf = new TezConfiguration();
+ conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG");
+
+ LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);
+
+ List<String> expectedCommands = new LinkedList<String>();
+ expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+ expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
+ TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+ for (int i = 0 ; i < testInfo.numTasks ; i++) {
+ ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+ String javaOpts = containerContext.getJavaOpts();
+ assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+ for (String expectedCmd : expectedCommands) {
+ assertTrue(javaOpts.contains(expectedCmd));
+ }
+
+ Map<String, String> env = containerContext.getEnvironment();
+ String val = env.get(testInfo.envKey);
+ assertEquals(testInfo.envVal, val);
+ String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ assertNull(logEnvVal);
+ }
+ }
+
+ @Test(timeout = 5000)
+ public void testTaskSpecificLoggingOpts() {
+
+ String vertexName = "testvertex";
+ String customJavaOpts = "-Xmx128m";
+
+ Configuration conf = new TezConfiguration();
+ conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "INFO");
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG;org.apache.tez=INFO");
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);
+
+ LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);
+
+ // Expected command opts for regular tasks
+ List<String> expectedCommands = new LinkedList<String>();
+ expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+ expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "INFO" + "," +
+ TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+ for (int i = 3 ; i < testInfo.numTasks ; i++) {
+ ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+ String javaOpts = containerContext.getJavaOpts();
+
+ assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+ for (String expectedCmd : expectedCommands) {
+ assertTrue(javaOpts.contains(expectedCmd));
+ }
+
+ Map<String, String> env = containerContext.getEnvironment();
+ String val = env.get(testInfo.envKey);
+ assertEquals(testInfo.envVal, val);
+ String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ assertNull(logEnvVal);
+ }
+
+ // Expected command opts for instrumented tasks.
+ expectedCommands = new LinkedList<String>();
+ expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
+ expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," +
+ TezConstants.TEZ_CONTAINER_LOGGER_NAME);
+
+ for (int i = 0 ; i < 3 ; i++) {
+ ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
+ String javaOpts = containerContext.getJavaOpts();
+
+ assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
+ for (String expectedCmd : expectedCommands) {
+ assertTrue(javaOpts.contains(expectedCmd));
+ }
+
+ Map<String, String> env = containerContext.getEnvironment();
+ String val = env.get(testInfo.envKey);
+ assertEquals(testInfo.envVal, val);
+ String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
+ assertEquals("org.apache.tez=INFO", logEnvVal);
+ }
+ }
+
+ private static class LogTestInfoHolder {
+
+ final AppContext mockAppContext;
+ final DAG mockDag;
+ final VertexImpl vertex;
+ final DAGProtos.VertexPlan vertexPlan;
+
+ final int numTasks = 10;
+ final String initialJavaOpts = "initialJavaOpts";
+ final String envKey = "key1";
+ final String envVal = "val1";
+
+ LogTestInfoHolder(Configuration conf) {
+ this(conf, "testvertex");
+ }
+
+ LogTestInfoHolder(Configuration conf, String vertexName) {
+ mockAppContext = mock(AppContext.class);
+ mockDag = mock(DAG.class);
+ doReturn(new Credentials()).when(mockDag).getCredentials();
+ doReturn(mockDag).when(mockAppContext).getCurrentDAG();
+
+ vertexPlan = DAGProtos.VertexPlan.newBuilder()
+ .setName(vertexName)
+ .setTaskConfig(DAGProtos.PlanTaskConfiguration.newBuilder()
+ .setJavaOpts(initialJavaOpts)
+ .setNumTasks(numTasks)
+ .setMemoryMb(1024)
+ .setVirtualCores(1)
+ .setTaskModule("taskmodule")
+ .addEnvironmentSetting(DAGProtos.PlanKeyValuePair.newBuilder()
+ .setKey(envKey)
+ .setValue(envVal)
+ .build())
+ .build())
+ .setType(DAGProtos.PlanVertexType.NORMAL).build();
+
+ vertex =
+ new VertexImpl(TezVertexID.fromString("vertex_1418197758681_0001_1_00"), vertexPlan,
+ "testvertex", conf, mock(EventHandler.class), mock(TaskAttemptListener.class),
+ mock(Clock.class), mock(TaskHeartbeatHandler.class), false, mockAppContext,
+ VertexLocationHint.create(new LinkedList<TaskLocationHint>()), null,
+ new TaskSpecificLaunchCmdOption(conf), mock(StateChangeNotifier.class));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/599bb0fe/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java b/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
index aebad74..f22eed6 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java
@@ -18,6 +18,7 @@
package org.apache.tez.dag.utils;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -40,7 +41,8 @@ public class TestTaskSpecificLaunchCmdOption {
return new TaskSpecificLaunchCmdOption(conf);
}
- @Test
+
+ @Test(timeout = 5000)
public void testTaskSpecificJavaOptions() {
Random rnd = new Random();
Configuration conf = new Configuration();
@@ -238,7 +240,7 @@ public class TestTaskSpecificLaunchCmdOption {
rnd.nextInt(Integer.MAX_VALUE)));
}
- @Test
+ @Test(timeout = 5000)
public void testConfigOptions() {
Configuration conf = new Configuration();
TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption = getOptions(conf, "", "");
@@ -263,4 +265,44 @@ public class TestTaskSpecificLaunchCmdOption {
optionStr = taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v", 4);
assertTrue(optionStr.equals("dir=/tmp/v/4"));
}
+
+
+ @Test(timeout = 5000)
+ public void testTaskSpecificLogOptions() {
+ Configuration conf = new Configuration(false);
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[0,2,5]");
+ TaskSpecificLaunchCmdOption options;
+
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG;org.apache.tez=INFO");
+ options = new TaskSpecificLaunchCmdOption(conf);
+ assertTrue(options.hasModifiedLogProperties());
+ assertFalse(options.hasModifiedTaskLaunchOpts());
+ assertEquals(2, options.getTaskSpecificLogParams().length);
+
+ conf.unset(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL);
+ options = new TaskSpecificLaunchCmdOption(conf);
+ assertFalse(options.hasModifiedLogProperties());
+ assertFalse(options.hasModifiedTaskLaunchOpts());
+
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG");
+ options = new TaskSpecificLaunchCmdOption(conf);
+ assertTrue(options.hasModifiedLogProperties());
+ assertFalse(options.hasModifiedTaskLaunchOpts());
+ assertEquals(1, options.getTaskSpecificLogParams().length);
+ }
+
+ @Test (timeout=5000)
+ public void testTaskSpecificLogOptionsWithCommandOptions() {
+ Configuration conf = new Configuration(false);
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[0,2,5]");
+ TaskSpecificLaunchCmdOption options;
+
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG;org.apache.tez=INFO");
+ conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "-Xmx128m");
+ options = new TaskSpecificLaunchCmdOption(conf);
+ assertTrue(options.hasModifiedLogProperties());
+ assertTrue(options.hasModifiedTaskLaunchOpts());
+ String optionStr = options.getTaskSpecificOption("", "v", 0);
+ assertTrue(optionStr.equals("-Xmx128m"));
+ }
}