You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2013/11/22 21:51:09 UTC
svn commit: r1544672 - in
/hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/
hadoop-yarn/hadoop-ya...
Author: arp
Date: Fri Nov 22 20:51:06 2013
New Revision: 1544672
URL: http://svn.apache.org/r1544672
Log:
Merging r1544304 through r1544665 from trunk to branch HDFS-2832
Added:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java
- copied unchanged from r1544665, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Fri Nov 22 20:51:06 2013
@@ -170,6 +170,9 @@ Release 2.3.0 - UNRELEASED
YARN-1053. Diagnostic message from ContainerExitEvent is ignored in
ContainerImpl (Omkar Vinit Joshi via bikas)
+ YARN-1320. Fixed Distributed Shell application to respect custom log4j
+ properties file. (Xuan Gong via vinodkv)
+
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri Nov 22 20:51:06 2013
@@ -220,6 +220,9 @@ public class ApplicationMaster {
// Hardcoded path to shell script in launch container's local env
private final String ExecShellStringPath = "ExecShellScript.sh";
+ // Hardcoded path to custom log_properties
+ private final String log4jPath = "log4j.properties";
+
private final String shellCommandPath = "shellCommands";
private volatile boolean done;
@@ -327,6 +330,16 @@ public class ApplicationMaster {
"No args specified for application master to initialize");
}
+ //Check whether customer log4j.properties file exists
+ File customerLog4jFile = new File(log4jPath);
+ if (customerLog4jFile.exists()) {
+ try {
+ Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath);
+ } catch (Exception e) {
+ LOG.warn("Can not set up custom log4j properties. " + e);
+ }
+ }
+
if (cliParser.hasOption("help")) {
printUsage(opts);
return false;
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java Fri Nov 22 20:51:06 2013
@@ -166,6 +166,9 @@ public class Client {
private Options opts;
private final String shellCommandPath = "shellCommands";
+ // Hardcoded path to custom log_properties
+ private final String log4jPath = "log4j.properties";
+
/**
* @param args Command line arguments
*/
@@ -257,7 +260,16 @@ public class Client {
if (args.length == 0) {
throw new IllegalArgumentException("No args specified for client to initialize");
- }
+ }
+
+ if (cliParser.hasOption("log_properties")) {
+ String log4jPath = cliParser.getOptionValue("log_properties");
+ try {
+ Log4jPropertyHelper.updateLog4jConfiguration(Client.class, log4jPath);
+ } catch (Exception e) {
+ LOG.warn("Can not set up custom log4j properties. " + e);
+ }
+ }
if (cliParser.hasOption("help")) {
printUsage();
@@ -455,16 +467,16 @@ public class Client {
// Set the log4j properties if needed
if (!log4jPropFile.isEmpty()) {
Path log4jSrc = new Path(log4jPropFile);
- Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
+ String log4jPathSuffix = appName + "/" + appId.getId() + "/" + log4jPath;
+ Path log4jDst = new Path(fs.getHomeDirectory(), log4jPathSuffix);
fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
- LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
- log4jRsrc.setType(LocalResourceType.FILE);
- log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
- log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
- log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
- log4jRsrc.setSize(log4jFileStatus.getLen());
- localResources.put("log4j.properties", log4jRsrc);
+ LocalResource log4jRsrc =
+ LocalResource.newInstance(
+ ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()),
+ LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
+ log4jFileStatus.getLen(), log4jFileStatus.getModificationTime());
+ localResources.put(log4jPath, log4jRsrc);
}
// The shell script has to be made available on the final container(s)
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Fri Nov 22 20:51:06 2013
@@ -25,6 +25,7 @@ import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.PrintWriter;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@@ -174,6 +175,68 @@ public class TestDistributedShell {
}
@Test(timeout=90000)
+ public void testDSShellWithCustomLogPropertyFile() throws Exception {
+ final File basedir =
+ new File("target", TestDistributedShell.class.getName());
+ final File tmpDir = new File(basedir, "tmpDir");
+ tmpDir.mkdirs();
+ final File customLogProperty = new File(tmpDir, "custom_log4j.properties");
+ if (customLogProperty.exists()) {
+ customLogProperty.delete();
+ }
+ if(!customLogProperty.createNewFile()) {
+ Assert.fail("Can not create custom log4j property file.");
+ }
+ PrintWriter fileWriter = new PrintWriter(customLogProperty);
+ // set the output to DEBUG level
+ fileWriter.write("log4j.rootLogger=debug,stdout");
+ fileWriter.close();
+ String[] args = {
+ "--jar",
+ APPMASTER_JAR,
+ "--num_containers",
+ "3",
+ "--shell_command",
+ "echo",
+ "--shell_args",
+ "HADOOP",
+ "--log_properties",
+ customLogProperty.getAbsolutePath(),
+ "--master_memory",
+ "512",
+ "--master_vcores",
+ "2",
+ "--container_memory",
+ "128",
+ "--container_vcores",
+ "1"
+ };
+
+ //Before run the DS, the default the log level is INFO
+ final Log LOG_Client =
+ LogFactory.getLog(Client.class);
+ Assert.assertTrue(LOG_Client.isInfoEnabled());
+ Assert.assertFalse(LOG_Client.isDebugEnabled());
+ final Log LOG_AM = LogFactory.getLog(ApplicationMaster.class);
+ Assert.assertTrue(LOG_AM.isInfoEnabled());
+ Assert.assertFalse(LOG_AM.isDebugEnabled());
+
+ LOG.info("Initializing DS Client");
+ final Client client =
+ new Client(new Configuration(yarnCluster.getConfig()));
+ boolean initSuccess = client.init(args);
+ Assert.assertTrue(initSuccess);
+ LOG.info("Running DS Client");
+ boolean result = client.run();
+ LOG.info("Client run completed. Result=" + result);
+ Assert.assertTrue(verifyContainerLog(3, null, true, "DEBUG") > 10);
+ //After DS is finished, the log level should be DEBUG
+ Assert.assertTrue(LOG_Client.isInfoEnabled());
+ Assert.assertTrue(LOG_Client.isDebugEnabled());
+ Assert.assertTrue(LOG_AM.isInfoEnabled());
+ Assert.assertTrue(LOG_AM.isDebugEnabled());
+ }
+
public void testDSShellWithCommands() throws Exception {
String[] args = {