You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/01/07 23:41:43 UTC
[26/50] [abbrv] hadoop git commit: MAPREDUCE-6577. MR AM unable to
load native library without MR_AM_ADMIN_USER_ENV set (sjlee)
MAPREDUCE-6577. MR AM unable to load native library without MR_AM_ADMIN_USER_ENV set (sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6f16118
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6f16118
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6f16118
Branch: refs/heads/HDFS-1312
Commit: f6f16118d38fcfe3b724f05fad752cb223f441ec
Parents: dec8fed
Author: Sangjin Lee <sj...@apache.org>
Authored: Tue Jan 5 15:22:50 2016 -0800
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Jan 5 15:22:50 2016 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../apache/hadoop/mapreduce/MRJobConfig.java | 21 ++++--
.../org/apache/hadoop/mapred/YARNRunner.java | 5 +-
.../hadoop/mapred/TestMiniMRChildTask.java | 68 ++++++++++----------
.../apache/hadoop/mapred/TestYARNRunner.java | 47 +++++++++++---
hadoop-project/pom.xml | 1 +
6 files changed, 95 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6f16118/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index d04ce5d..ce22824 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -315,6 +315,9 @@ Release 2.9.0 - UNRELEASED
BUG FIXES
+ MAPREDUCE-6577. MR AM unable to load native library without
+ MR_AM_ADMIN_USER_ENV set (sjlee)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6f16118/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 3d1e841..c98746a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -740,6 +740,16 @@ public interface MRJobConfig {
public static final String MR_AM_ADMIN_USER_ENV =
MR_AM_PREFIX + "admin.user.env";
+ // although the AM admin user env default should be the same as the task user
+ // env default, there are problems in making it work on Windows currently
+ // MAPREDUCE-6588 should address the issue and set it to a proper non-empty
+ // value
+ public static final String DEFAULT_MR_AM_ADMIN_USER_ENV =
+ Shell.WINDOWS ?
+ "" :
+ "LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
+ "/lib/native";
+
public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile";
public static final boolean DEFAULT_MR_AM_PROFILE = false;
public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX
@@ -763,10 +773,13 @@ public interface MRJobConfig {
public static final String MAPRED_ADMIN_USER_ENV =
"mapreduce.admin.user.env";
- public final String DEFAULT_MAPRED_ADMIN_USER_ENV =
- Shell.WINDOWS ?
- "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin":
- "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native";
+ // the "%...%" macros can be expanded prematurely and are probably not OK
+ // this should be addressed by MAPREDUCE-6588
+ public static final String DEFAULT_MAPRED_ADMIN_USER_ENV =
+ Shell.WINDOWS ?
+ "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" :
+ "LD_LIBRARY_PATH=" + Apps.crossPlatformify("HADOOP_COMMON_HOME") +
+ "/lib/native";
public static final String WORKDIR = "work";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6f16118/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index f15b5c1..fe631de 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -472,13 +472,14 @@ public class YARNRunner implements ClientProtocol {
conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL,
MRJobConfig.DEFAULT_SHELL));
- // Add the container working directory at the front of LD_LIBRARY_PATH
+ // Add the container working directory in front of LD_LIBRARY_PATH
MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(),
MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
// Setup the environment variables for Admin first
MRApps.setEnvFromInputString(environment,
- conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV), conf);
+ conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV,
+ MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV), conf);
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ENV), conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6f16118/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
index 59aa00f..74be8a7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
@@ -395,56 +395,56 @@ public class TestMiniMRChildTask {
/**
* To test OS dependent setting of default execution path for a MapRed task.
* Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -
- * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH - for
- * Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
+ * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH -
+ * for Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
* LD_LIBRARY_PATH
*/
@Test
public void testMapRedExecutionEnv() {
- // test if the env variable can be set
- try {
- // Application environment
- Map<String, String> environment = new HashMap<String, String>();
- String setupHadoopHomeCommand = Shell.WINDOWS ?
- "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home" :
- "HADOOP_COMMON_HOME=/fake/path/to/hadoop/common/home";
- MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
-
- // Add the env variables passed by the admin
- MRApps.setEnvFromInputString(environment, conf.get(
- MRJobConfig.MAPRED_ADMIN_USER_ENV,
- MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
-
- String executionPaths = environment.get(
- Shell.WINDOWS ? "PATH" : "LD_LIBRARY_PATH");
- String toFind = Shell.WINDOWS ?
- "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin" :
- "/fake/path/to/hadoop/common/home/lib/native";
-
- // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
- assertTrue("execution path does not include the hadoop lib location "
- + toFind, executionPaths.contains(toFind));
- } catch (Exception e) {
- e.printStackTrace();
- fail("Exception in testing execution environment for MapReduce task");
- tearDown();
+ // for windows, test if the env variable can be set
+ // this may be removed as part of MAPREDUCE-6588
+ if (Shell.WINDOWS) {
+ try {
+ // Application environment
+ Map<String, String> environment = new HashMap<String, String>();
+ String setupHadoopHomeCommand =
+ "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home";
+ MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
+
+ // Add the env variables passed by the admin
+ MRApps.setEnvFromInputString(environment, conf.get(
+ MRJobConfig.MAPRED_ADMIN_USER_ENV,
+ MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
+
+ String executionPaths = environment.get("PATH");
+ String toFind =
+ "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin";
+
+ // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
+ assertTrue("execution path does not include the hadoop lib location "
+ + toFind, executionPaths.contains(toFind));
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("Exception in testing execution environment for MapReduce task");
+ tearDown();
+ }
}
-
+
// now launch a mapreduce job to ensure that the child
// also gets the configured setting for hadoop lib
try {
-
- JobConf conf = new JobConf(mr.getConfig());
+
+ JobConf conf = new JobConf(mr.getConfig());
// initialize input, output directories
Path inDir = new Path("input");
Path outDir = new Path("output");
String input = "The input";
-
+
// set config to use the ExecutionEnvCheckMapClass map class
configure(conf, inDir, outDir, input,
ExecutionEnvCheckMapClass.class, IdentityReducer.class);
launchTest(conf, inDir, outDir, input);
-
+
} catch(Exception e) {
e.printStackTrace();
fail("Exception in testing propagation of env setting to child task");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6f16118/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 3293a9d..0e34042 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -570,16 +571,34 @@ public class TestYARNRunner extends TestCase {
}
@Test
- public void testAMStandardEnv() throws Exception {
+ public void testAMStandardEnvWithDefaultLibPath() throws Exception {
+ testAMStandardEnv(false);
+ }
+
+ @Test
+ public void testAMStandardEnvWithCustomLibPath() throws Exception {
+ testAMStandardEnv(true);
+ }
+
+ private void testAMStandardEnv(boolean customLibPath) throws Exception {
+ // the Windows behavior is different and this test currently doesn't really
+ // apply
+ // MAPREDUCE-6588 should revisit this test
+ if (Shell.WINDOWS) {
+ return;
+ }
+
final String ADMIN_LIB_PATH = "foo";
final String USER_LIB_PATH = "bar";
final String USER_SHELL = "shell";
JobConf jobConf = new JobConf();
+ String pathKey = Environment.LD_LIBRARY_PATH.name();
- jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "LD_LIBRARY_PATH=" +
- ADMIN_LIB_PATH);
- jobConf.set(MRJobConfig.MR_AM_ENV, "LD_LIBRARY_PATH="
- + USER_LIB_PATH);
+ if (customLibPath) {
+ jobConf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, pathKey + "=" +
+ ADMIN_LIB_PATH);
+ jobConf.set(MRJobConfig.MR_AM_ENV, pathKey + "=" + USER_LIB_PATH);
+ }
jobConf.set(MRJobConfig.MAPRED_ADMIN_USER_SHELL, USER_SHELL);
YARNRunner yarnRunner = new YARNRunner(jobConf);
@@ -589,15 +608,23 @@ public class TestYARNRunner extends TestCase {
// make sure PWD is first in the lib path
ContainerLaunchContext clc = appSubCtx.getAMContainerSpec();
Map<String, String> env = clc.getEnvironment();
- String libPath = env.get(Environment.LD_LIBRARY_PATH.name());
- assertNotNull("LD_LIBRARY_PATH not set", libPath);
+ String libPath = env.get(pathKey);
+ assertNotNull(pathKey + " not set", libPath);
String cps = jobConf.getBoolean(
MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
- assertEquals("Bad AM LD_LIBRARY_PATH setting",
- MRApps.crossPlatformifyMREnv(conf, Environment.PWD)
- + cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH, libPath);
+ String expectedLibPath =
+ MRApps.crossPlatformifyMREnv(conf, Environment.PWD);
+ if (customLibPath) {
+ // append admin libpath and user libpath
+ expectedLibPath += cps + ADMIN_LIB_PATH + cps + USER_LIB_PATH;
+ } else {
+ expectedLibPath += cps +
+ MRJobConfig.DEFAULT_MR_AM_ADMIN_USER_ENV.substring(
+ pathKey.length() + 1);
+ }
+ assertEquals("Bad AM " + pathKey + " setting", expectedLibPath, libPath);
// make sure SHELL is set
String shell = env.get(Environment.SHELL.name());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6f16118/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index bb844f4..36d735a 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1133,6 +1133,7 @@
<forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
<argLine>${maven-surefire-plugin.argLine}</argLine>
<environmentVariables>
+ <HADOOP_COMMON_HOME>${hadoop.common.build.dir}</HADOOP_COMMON_HOME>
<!-- HADOOP_HOME required for tests on Windows to find winutils -->
<HADOOP_HOME>${hadoop.common.build.dir}</HADOOP_HOME>
<LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib</LD_LIBRARY_PATH>