You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2013/01/09 07:31:36 UTC
svn commit: r1430707 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/
hadoop-...
Author: vinodkv
Date: Wed Jan 9 06:31:36 2013
New Revision: 1430707
URL: http://svn.apache.org/viewvc?rev=1430707&view=rev
Log:
MAPREDUCE-4810. Added new admin command options for MR AM. Contributed by Jerry Chen.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1430707&r1=1430706&r2=1430707&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Jan 9 06:31:36 2013
@@ -175,6 +175,9 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-4520. Added support for MapReduce applications to request for
CPU cores along-with memory post YARN-2. (acmurthy)
+ MAPREDUCE-4810. Added new admin command options for MR AM. (Jerry Chen via
+ vinodkv)
+
IMPROVEMENTS
MAPREDUCE-3678. The Map tasks logs should have the value of input
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1430707&r1=1430706&r2=1430707&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Wed Jan 9 06:31:36 2013
@@ -365,6 +365,11 @@ public interface MRJobConfig {
MR_AM_PREFIX+"command-opts";
public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1024m";
+ /** Admin command opts passed to the MR app master.*/
+ public static final String MR_AM_ADMIN_COMMAND_OPTS =
+ MR_AM_PREFIX+"admin-command-opts";
+ public static final String DEFAULT_MR_AM_ADMIN_COMMAND_OPTS = "";
+
/** Root Logging level passed to the MR app master.*/
public static final String MR_AM_LOG_LEVEL =
MR_AM_PREFIX+"log.level";
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1430707&r1=1430706&r2=1430707&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Wed Jan 9 06:31:36 2013
@@ -875,6 +875,20 @@
</property>
<property>
+ <name>yarn.app.mapreduce.am.admin-command-opts</name>
+ <value></value>
+ <description>Java opts for the MR App Master processes for admin purposes.
+ It will appears before the opts set by yarn.app.mapreduce.am.command-opts and
+ thus its options can be overridden user.
+
+ Usage of -Djava.library.path can cause programs to no longer function if
+ hadoop native libraries are used. These values should instead be set as part
+ of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
+ mapreduce.reduce.env config settings.
+ </description>
+</property>
+
+<property>
<name>yarn.app.mapreduce.am.job.task.listener.thread-count</name>
<value>30</value>
<description>The number of threads used to handle RPC calls in the
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1430707&r1=1430706&r2=1430707&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Wed Jan 9 06:31:36 2013
@@ -394,6 +394,11 @@ public class YARNRunner implements Clien
MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
+ // Add AM admin command opts before user command opts
+ // so that it can be overridden by user
+ vargs.add(conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS));
+
vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java?rev=1430707&r1=1430706&r2=1430707&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java Wed Jan 9 06:31:36 2013
@@ -29,6 +29,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.List;
import junit.framework.TestCase;
@@ -39,11 +40,13 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.ClientCache;
import org.apache.hadoop.mapred.ClientServiceDelegate;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.ResourceMgrDelegate;
import org.apache.hadoop.mapred.YARNRunner;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobPriority;
import org.apache.hadoop.mapreduce.JobStatus.State;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.ClientRMProtocol;
@@ -65,6 +68,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -240,4 +244,68 @@ public class TestYARNRunner extends Test
delegate.getQueueAclsForCurrentUser();
verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
}
+
+ @Test
+ public void testAMAdminCommandOpts() throws Exception {
+ JobConf jobConf = new JobConf();
+
+ jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true");
+ jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m");
+
+ YARNRunner yarnRunner = new YARNRunner(jobConf);
+
+ File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
+ OutputStream out = new FileOutputStream(jobxml);
+ conf.writeXml(out);
+ out.close();
+
+ File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
+ out = new FileOutputStream(jobsplit);
+ out.close();
+
+ File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO);
+ out = new FileOutputStream(jobsplitmetainfo);
+ out.close();
+
+ File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE);
+ out = new FileOutputStream(appTokens);
+ out.close();
+
+ ApplicationSubmissionContext submissionContext =
+ yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials());
+
+ ContainerLaunchContext containerSpec = submissionContext.getAMContainerSpec();
+ List<String> commands = containerSpec.getCommands();
+
+ int index = 0;
+ int adminIndex = 0;
+ int adminPos = -1;
+ int userIndex = 0;
+ int userPos = -1;
+
+ for(String command : commands) {
+ if(command != null) {
+ adminPos = command.indexOf("-Djava.net.preferIPv4Stack=true");
+ if(adminPos >= 0)
+ adminIndex = index;
+
+ userPos = command.indexOf("-Xmx1024m");
+ if(userPos >= 0)
+ userIndex = index;
+ }
+
+ index++;
+ }
+
+ // Check both admin java opts and user java opts are in the commands
+ assertTrue("AM admin command opts not in the commands.", adminPos > 0);
+ assertTrue("AM user command opts not in the commands.", userPos > 0);
+
+ // Check the admin java opts is before user java opts in the commands
+ if(adminIndex == userIndex) {
+ assertTrue("AM admin command opts is after user command opts.", adminPos < userPos);
+ } else {
+ assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex);
+ }
+ }
}