You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by je...@apache.org on 2013/01/18 23:47:14 UTC
svn commit: r1435394 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resour...
Author: jeagles
Date: Fri Jan 18 22:47:14 2013
New Revision: 1435394
URL: http://svn.apache.org/viewvc?rev=1435394&view=rev
Log:
svn merge -c 1435386 FIXES: MAPREDUCE-4458. Warn if java.library.path is used for AM or Task (Robert Parker via jeagles)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1435394&r1=1435393&r2=1435394&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri Jan 18 22:47:14 2013
@@ -495,6 +495,9 @@ Release 0.23.7 - UNRELEASED
BUG FIXES
+ MAPREDUCE-4458. Warn if java.library.path is used for AM or Task
+ (Robert Parker via jeagles)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1435394&r1=1435393&r2=1435394&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Fri Jan 18 22:47:14 2013
@@ -527,6 +527,9 @@ public interface MRJobConfig {
public static final String MR_AM_ENV =
MR_AM_PREFIX + "env";
+ public static final String MR_AM_ADMIN_USER_ENV =
+ MR_AM_PREFIX + "admin.user.env";
+
public static final String MAPRED_MAP_ADMIN_JAVA_OPTS =
"mapreduce.admin.map.child.java.opts";
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1435394&r1=1435393&r2=1435394&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Fri Jan 18 22:47:14 2013
@@ -1282,6 +1282,17 @@
</property>
<property>
+ <name>yarn.app.mapreduce.am.admin.user.env</name>
+ <value></value>
+ <description> Environment variables for the MR App Master
+ processes for admin purposes. These values are set first and can be
+ overridden by the user env (yarn.app.mapreduce.am.env) Example :
+ 1) A=foo This will set the env variable A to foo
+ 2) B=$B:c This is inherit app master's B env variable.
+ </description>
+</property>
+
+<property>
<name>yarn.app.mapreduce.am.command-opts</name>
<value>-Xmx1024m</value>
<description>Java opts for the MR App Master processes.
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1435394&r1=1435393&r2=1435394&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Fri Jan 18 22:47:14 2013
@@ -101,7 +101,7 @@ public class YARNRunner implements Clien
private Configuration conf;
private final FileContext defaultFileContext;
- /* usually is false unless the jobclient getdelegation token is
+ /* usually is false unless the jobclient get delegation token is
* called. This is a hack wherein we do return a token from RM
* on getDelegationtoken but due to the restricted api on jobclient
* we just add a job history DT token when submitting a job.
@@ -165,12 +165,12 @@ public class YARNRunner implements Clien
@Override
public TaskTrackerInfo[] getActiveTrackers() throws IOException,
InterruptedException {
- return resMgrDelegate.getActiveTrackers();
+ return resMgrDelegate.getActiveTrackers();
}
@Override
public JobStatus[] getAllJobs() throws IOException, InterruptedException {
- return resMgrDelegate.getAllJobs();
+ return resMgrDelegate.getAllJobs();
}
@Override
@@ -394,14 +394,31 @@ public class YARNRunner implements Clien
MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
+ // Check for Java Lib Path usage in MAP and REDUCE configs
+ warnForJavaLibPath(conf.get(MRJobConfig.MAP_JAVA_OPTS,""), "map",
+ MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.MAP_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,""), "map",
+ MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce",
+ MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce",
+ MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
+
// Add AM admin command opts before user command opts
// so that it can be overridden by user
- vargs.add(conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
- MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS));
+ String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
+ warnForJavaLibPath(mrAppMasterAdminOptions, "app master",
+ MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, MRJobConfig.MR_AM_ADMIN_USER_ENV);
+ vargs.add(mrAppMasterAdminOptions);
+
+ // Add AM user command opts
+ String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
+ warnForJavaLibPath(mrAppMasterUserOptions, "app master",
+ MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
+ vargs.add(mrAppMasterUserOptions);
- vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
- MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));
-
vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDOUT);
@@ -425,6 +442,9 @@ public class YARNRunner implements Clien
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
+ // Setup the environment variables for Admin first
+ MRApps.setEnvFromInputString(environment,
+ conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ENV));
@@ -582,4 +602,15 @@ public class YARNRunner implements Clien
throws IOException {
return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID);
}
+
+ private static void warnForJavaLibPath(String opts, String component,
+ String javaConf, String envConf) {
+ if (opts != null && opts.contains("-Djava.library.path")) {
+ LOG.warn("Usage of -Djava.library.path in " + javaConf + " can cause " +
+ "programs to no longer function if hadoop native libraries " +
+ "are used. These values should be set as part of the " +
+ "LD_LIBRARY_PATH in the " + component + " JVM env using " +
+ envConf + " config settings.");
+ }
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java?rev=1435394&r1=1435393&r2=1435394&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java Fri Jan 18 22:47:14 2013
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -83,6 +84,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.log4j.Appender;
+import org.apache.log4j.Layout;
+import org.apache.log4j.Logger;
+import org.apache.log4j.SimpleLayout;
+import org.apache.log4j.WriterAppender;
import org.junit.Before;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
@@ -189,7 +195,7 @@ public class TestYARNRunner extends Test
@Test
public void testResourceMgrDelegate() throws Exception {
- /* we not want a mock of resourcemgr deleagte */
+ /* we not want a mock of resource mgr delegate */
final ClientRMProtocol clientRMProtocol = mock(ClientRMProtocol.class);
ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) {
@Override
@@ -256,6 +262,9 @@ public class TestYARNRunner extends Test
@Test
public void testHistoryServerToken() throws Exception {
+ //Set the master principal in the config
+ conf.set(YarnConfiguration.RM_PRINCIPAL,"foo@LOCAL");
+
final String masterPrincipal = Master.getMasterPrincipal(conf);
final MRClientProtocol hsProxy = mock(MRClientProtocol.class);
@@ -265,7 +274,7 @@ public class TestYARNRunner extends Test
GetDelegationTokenRequest request =
(GetDelegationTokenRequest)invocation.getArguments()[0];
// check that the renewer matches the cluster's RM principal
- assertEquals(request.getRenewer(), masterPrincipal);
+ assertEquals(masterPrincipal, request.getRenewer() );
DelegationToken token =
recordFactory.newRecordInstance(DelegationToken.class);
@@ -357,4 +366,53 @@ public class TestYARNRunner extends Test
assertTrue("AM admin command opts is after user command opts.", adminIndex < userIndex);
}
}
+ @Test
+ public void testWarnCommandOpts() throws Exception {
+ Logger logger = Logger.getLogger(YARNRunner.class);
+
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ Layout layout = new SimpleLayout();
+ Appender appender = new WriterAppender(layout, bout);
+ logger.addAppender(appender);
+
+ JobConf jobConf = new JobConf();
+
+ jobConf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, "-Djava.net.preferIPv4Stack=true -Djava.library.path=foo");
+ jobConf.set(MRJobConfig.MR_AM_COMMAND_OPTS, "-Xmx1024m -Djava.library.path=bar");
+
+ YARNRunner yarnRunner = new YARNRunner(jobConf);
+
+ File jobxml = new File(testWorkDir, MRJobConfig.JOB_CONF_FILE);
+ OutputStream out = new FileOutputStream(jobxml);
+ conf.writeXml(out);
+ out.close();
+
+ File jobsplit = new File(testWorkDir, MRJobConfig.JOB_SPLIT);
+ out = new FileOutputStream(jobsplit);
+ out.close();
+
+ File jobsplitmetainfo = new File(testWorkDir, MRJobConfig.JOB_SPLIT_METAINFO);
+ out = new FileOutputStream(jobsplitmetainfo);
+ out.close();
+
+ File appTokens = new File(testWorkDir, MRJobConfig.APPLICATION_TOKENS_FILE);
+ out = new FileOutputStream(appTokens);
+ out.close();
+
+ @SuppressWarnings("unused")
+ ApplicationSubmissionContext submissionContext =
+ yarnRunner.createApplicationSubmissionContext(jobConf, testWorkDir.toString(), new Credentials());
+
+ String logMsg = bout.toString();
+ assertTrue(logMsg.contains("WARN - Usage of -Djava.library.path in " +
+ "yarn.app.mapreduce.am.admin-command-opts can cause programs to no " +
+ "longer function if hadoop native libraries are used. These values " +
+ "should be set as part of the LD_LIBRARY_PATH in the app master JVM " +
+ "env using yarn.app.mapreduce.am.admin.user.env config settings."));
+ assertTrue(logMsg.contains("WARN - Usage of -Djava.library.path in " +
+ "yarn.app.mapreduce.am.command-opts can cause programs to no longer " +
+ "function if hadoop native libraries are used. These values should " +
+ "be set as part of the LD_LIBRARY_PATH in the app master JVM env " +
+ "using yarn.app.mapreduce.am.env config settings."));
+ }
}