You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2014/10/15 19:56:49 UTC
git commit: MAPREDUCE-5970. Provide a boolean switch to enable MR-AM
profiling. Contributed by Gera Shegalov (cherry picked from commit
f19771a24c2f90982cf6dec35889836a6146c968)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 e68fef4d8 -> 025b6c13e
MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling. Contributed by Gera Shegalov
(cherry picked from commit f19771a24c2f90982cf6dec35889836a6146c968)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/025b6c13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/025b6c13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/025b6c13
Branch: refs/heads/branch-2
Commit: 025b6c13e930831c40eb07a15131b69fa625a188
Parents: e68fef4
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Oct 15 17:50:31 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Oct 15 17:51:53 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/mapred/JobConf.java | 3 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 11 ++++++-
.../org/apache/hadoop/mapred/YARNRunner.java | 15 +++++++--
.../apache/hadoop/mapred/TestYARNRunner.java | 32 +++++++++++++++++++-
5 files changed, 58 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/025b6c13/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index b6be1c3..1ede424 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -55,6 +55,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6072. Remove INSTALL document (Akira AJISAKA via aw)
+ MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling (Gera
+ Shegalov via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/025b6c13/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index de78e20..03f1160 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -1643,8 +1643,7 @@ public class JobConf extends Configuration {
*/
public String getProfileParams() {
return get(JobContext.TASK_PROFILE_PARAMS,
- "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y," +
- "verbose=n,file=%s");
+ MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/025b6c13/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 33c0461..12bf6df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -179,6 +179,10 @@ public interface MRJobConfig {
public static final String TASK_PROFILE_PARAMS = "mapreduce.task.profile.params";
+ public static final String DEFAULT_TASK_PROFILE_PARAMS =
+ "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,"
+ + "verbose=n,file=%s";
+
public static final String NUM_MAP_PROFILES = "mapreduce.task.profile.maps";
public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces";
@@ -603,7 +607,12 @@ public interface MRJobConfig {
public static final String MR_AM_ADMIN_USER_ENV =
MR_AM_PREFIX + "admin.user.env";
-
+
+ public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
+ public static final boolean DEFAULT_MR_AM_PROFILE = false;
+ public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
+ + "profile.params";
+
public static final String MAPRED_MAP_ADMIN_JAVA_OPTS =
"mapreduce.admin.map.child.java.opts";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/025b6c13/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 9419d03..d3b80f3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -408,7 +408,7 @@ public class YARNRunner implements ClientProtocol {
warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce",
MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV);
warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce",
- MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
+ MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
// Add AM admin command opts before user command opts
// so that it can be overridden by user
@@ -424,7 +424,18 @@ public class YARNRunner implements ClientProtocol {
warnForJavaLibPath(mrAppMasterUserOptions, "app master",
MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
vargs.add(mrAppMasterUserOptions);
-
+
+ if (jobConf.getBoolean(MRJobConfig.MR_AM_PROFILE,
+ MRJobConfig.DEFAULT_MR_AM_PROFILE)) {
+ final String profileParams = jobConf.get(MRJobConfig.MR_AM_PROFILE_PARAMS,
+ MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS);
+ if (profileParams != null) {
+ vargs.add(String.format(profileParams,
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR
+ + TaskLog.LogName.PROFILE));
+ }
+ }
+
vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDOUT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/025b6c13/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 38a79fb..2567785 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -29,7 +29,6 @@ import static org.mockito.Mockito.when;
import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -115,6 +114,11 @@ public class TestYARNRunner extends TestCase {
private static final Log LOG = LogFactory.getLog(TestYARNRunner.class);
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ // prefix before <LOG_DIR>/profile.out
+ private static final String PROFILE_PARAMS =
+ MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.substring(0,
+ MRJobConfig.DEFAULT_TASK_PROFILE_PARAMS.lastIndexOf("%"));
+
private YARNRunner yarnRunner;
private ResourceMgrDelegate resourceMgrDelegate;
private YarnConfiguration conf;
@@ -423,6 +427,8 @@ public class TestYARNRunner extends TestCase {
for(String command : commands) {
if(command != null) {
+ assertFalse("Profiler should be disabled by default",
+ command.contains(PROFILE_PARAMS));
adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true");
if(adminPos >= 0)
adminIndex = index;
@@ -479,6 +485,30 @@ public class TestYARNRunner extends TestCase {
"using yarn.app.mapreduce.am.env config settings."));
}
+ @Test(timeout=20000)
+ public void testAMProfiler() throws Exception {
+ JobConf jobConf = new JobConf();
+
+ jobConf.setBoolean(MRJobConfig.MR_AM_PROFILE, true);
+
+ YARNRunner yarnRunner = new YARNRunner(jobConf);
+
+ ApplicationSubmissionContext submissionContext =
+ buildSubmitContext(yarnRunner, jobConf);
+
+ ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
+ List<String> commands = containerSpec.getCommands();
+
+ for(String command : commands) {
+ if (command != null) {
+ if (command.contains(PROFILE_PARAMS)) {
+ return;
+ }
+ }
+ }
+ throw new IllegalStateException("Profiler opts not found!");
+ }
+
@Test
public void testAMStandardEnv() throws Exception {
final String ADMIN_LIB_PATH = "foo";