You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ge...@apache.org on 2015/06/04 20:19:29 UTC
hadoop git commit: MAPREDUCE-5232. Add a configuration to be able to
log classpath and other system properties on mapreduce JVMs startup.
Contributed by Sangjin Lee.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2e99210e4 -> 6044b5e5f
MAPREDUCE-5232. Add a configuration to be able to log classpath and other system properties on mapreduce JVMs startup. Contributed by Sangjin Lee.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1482643 13f79535-47bb-0310-9956-ffa450edef68
(cherry picked from commit 4d8e350750748b919ee2158690a44cd9fd80dcae)
Conflicts:
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6044b5e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6044b5e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6044b5e5
Branch: refs/heads/branch-2
Commit: 6044b5e5f6402ddcc163d9087af6228f24291fd2
Parents: 2e99210
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Tue May 14 23:43:53 2013 +0000
Committer: Gera Shegalov <ge...@apache.org>
Committed: Thu Jun 4 11:15:30 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../org/apache/hadoop/mapred/YarnChild.java | 6 ++++
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 6 ++++
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 31 ++++++++++++++++++++
.../hadoop/mapreduce/v2/util/TestMRApps.java | 21 ++++++++++++-
.../apache/hadoop/mapreduce/MRJobConfig.java | 5 ++++
.../src/main/resources/mapred-default.xml | 6 ++++
7 files changed, 77 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index cf40293..d215fc2 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -189,6 +189,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6382. Don't escape HTML links in Diagnostics in JHS job overview.
(Siqi Li via gera)
+ MAPREDUCE-5232. Add a configuration to be able to log classpath and other
+ system properties on mapreduce JVMs startup. (Sangjin Lee via vinodkv)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index 7f3111f..ea9733c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -141,6 +141,12 @@ class YarnChild {
// Create the job-conf and set credentials
configureTask(job, task, credentials, jt);
+ // log the system properties
+ String systemPropsToLog = MRApps.getSystemPropertiesToLog(job);
+ if (systemPropsToLog != null) {
+ LOG.info(systemPropsToLog);
+ }
+
// Initiate Java VM metrics
JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId());
childUGI = UserGroupInformation.createRemoteUser(System
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 99c5822..942e25b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -1495,6 +1495,12 @@ public class MRAppMaster extends CompositeService {
conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
MRWebAppUtil.initialize(conf);
+ // log the system properties
+ String systemPropsToLog = MRApps.getSystemPropertiesToLog(conf);
+ if (systemPropsToLog != null) {
+ LOG.info(systemPropsToLog);
+ }
+
String jobUserName = System
.getenv(ApplicationConstants.Environment.USER.name());
conf.set(MRJobConfig.USER_NAME, jobUserName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index 7d1a1b5..40ba613 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -733,4 +733,35 @@ public class MRApps extends Apps {
MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM);
return crossPlatform ? env.$$() : env.$();
}
+
+ /**
+ * Return lines for system property keys and values per configuration.
+ *
+ * @return the formatted string for the system property lines or null if no
+ * properties are specified.
+ */
+ public static String getSystemPropertiesToLog(Configuration conf) {
+ String key = conf.get(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG,
+ MRJobConfig.DEFAULT_MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG);
+ if (key != null) {
+ key = key.trim(); // trim leading and trailing whitespace from the config
+ if (!key.isEmpty()) {
+ String[] props = key.split(",");
+ if (props.length > 0) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\n/************************************************************\n");
+ sb.append("[system properties]\n");
+ for (String prop: props) {
+ prop = prop.trim(); // trim leading and trailing whitespace
+ if (!prop.isEmpty()) {
+ sb.append(prop).append(": ").append(System.getProperty(prop)).append('\n');
+ }
+ }
+ sb.append("************************************************************/");
+ return sb.toString();
+ }
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
index 2e77f5e..b233c84 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
@@ -480,7 +480,26 @@ public class TestMRApps {
}
public void initialize(URI name, Configuration conf) throws IOException {}
}
-
+
+ @Test
+ public void testLogSystemProperties() throws Exception {
+ Configuration conf = new Configuration();
+ // test no logging
+ conf.set(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG, " ");
+ String value = MRApps.getSystemPropertiesToLog(conf);
+ assertNull(value);
+
+ // test logging of selected keys
+ String classpath = "java.class.path";
+ String os = "os.name";
+ String version = "java.version";
+ conf.set(MRJobConfig.MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG, classpath + ", " + os);
+ value = MRApps.getSystemPropertiesToLog(conf);
+ assertNotNull(value);
+ assertTrue(value.contains(classpath));
+ assertTrue(value.contains(os));
+ assertFalse(value.contains(version));
+ }
@Test
public void testTaskStateUI() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 61d1ce7..202f860 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -205,6 +205,11 @@ public interface MRJobConfig {
public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes";
+ public static final String MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG = "mapreduce.jvm.system-properties-to-log";
+ public static final String DEFAULT_MAPREDUCE_JVM_SYSTEM_PROPERTIES_TO_LOG =
+ "os.name,os.version,java.home,java.runtime.version,java.vendor," +
+ "java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name";
+
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";
public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6044b5e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 1b109f8..9de2c4e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1981,6 +1981,12 @@
</description>
</property>
+<property>
+ <name>mapreduce.jvm.system-properties-to-log</name>
+ <value>os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name</value>
+ <description>Comma-delimited list of system properties to log on mapreduce JVM start</description>
+</property>
+
<!-- jobhistory properties -->
<property>