You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/06/10 03:35:41 UTC
[30/50] [abbrv] hadoop git commit: MAPREDUCE-6702.
TestMiniMRChildTask.testTaskEnv and TestMiniMRChildTask.testTaskOldEnv are
failing (ajisakaa via rkanter)
MAPREDUCE-6702. TestMiniMRChildTask.testTaskEnv and TestMiniMRChildTask.testTaskOldEnv are failing (ajisakaa via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/733f3f18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/733f3f18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/733f3f18
Branch: refs/heads/HDFS-7240
Commit: 733f3f18d5cf38cbae35146fbef8e16e35fdf5e1
Parents: 620325e
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 7 15:46:06 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Jun 7 15:46:06 2016 -0700
----------------------------------------------------------------------
.../src/site/markdown/SingleCluster.md.vm | 12 +-
.../java/org/apache/hadoop/mapred/JobConf.java | 6 -
.../hadoop/mapred/TestMiniMRChildTask.java | 233 +++----------------
3 files changed, 41 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/733f3f18/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
index 573ca32..4825e00 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
@@ -181,13 +181,23 @@ You can run a MapReduce job on YARN in a pseudo-distributed mode by setting a fe
The following instructions assume that 1. ~ 4. steps of [the above instructions](#Execution) are already executed.
-1. Configure parameters as follows:`etc/hadoop/mapred-site.xml`:
+1. Configure parameters as follows:
+
+ `etc/hadoop/mapred-site.xml`:
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
+ <property>
+ <name>mapreduce.admin.user.env</name>
+ <value>HADOOP_MAPRED_HOME=$HADOOP_COMMON_HOME</value>
+ </property>
+ <property>
+ <name>yarn.app.mapreduce.am.env</name>
+ <value>HADOOP_MAPRED_HOME=$HADOOP_COMMON_HOME</value>
+ </property>
</configuration>
`etc/hadoop/yarn-site.xml`:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/733f3f18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index 2cfce1f..f2b0aae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -294,8 +294,6 @@ public class JobConf extends Configuration {
* Example:
* <ul>
* <li> A=foo - This will set the env variable A to foo. </li>
- * <li> B=$X:c This is inherit tasktracker's X env variable on Linux. </li>
- * <li> B=%X%;c This is inherit tasktracker's X env variable on Windows. </li>
* </ul>
*
* @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or
@@ -314,8 +312,6 @@ public class JobConf extends Configuration {
* Example:
* <ul>
* <li> A=foo - This will set the env variable A to foo. </li>
- * <li> B=$X:c This is inherit tasktracker's X env variable on Linux. </li>
- * <li> B=%X%;c This is inherit tasktracker's X env variable on Windows. </li>
* </ul>
*/
public static final String MAPRED_MAP_TASK_ENV = JobContext.MAP_ENV;
@@ -330,8 +326,6 @@ public class JobConf extends Configuration {
* Example:
* <ul>
* <li> A=foo - This will set the env variable A to foo. </li>
- * <li> B=$X:c This is inherit tasktracker's X env variable on Linux. </li>
- * <li> B=%X%;c This is inherit tasktracker's X env variable on Windows. </li>
* </ul>
*/
public static final String MAPRED_REDUCE_TASK_ENV = JobContext.REDUCE_ENV;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/733f3f18/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
index cbeeccf..f690118 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
@@ -25,9 +25,7 @@ import static org.junit.Assert.fail;
import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
-import java.util.HashMap;
import java.util.Iterator;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -41,11 +39,9 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
-import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.Shell;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -111,29 +107,6 @@ public class TestMiniMRChildTask {
}
}
}
-
- /**
- * Map class which checks if hadoop lib location
- * is in the execution path
- */
- public static class ExecutionEnvCheckMapClass extends MapReduceBase
- implements Mapper<LongWritable, Text, Text, IntWritable> {
- public void map (LongWritable key, Text value,
- OutputCollector<Text, IntWritable> output,
- Reporter reporter) throws IOException {
- }
- public void configure(JobConf job) {
- String executionEnvPathVariable = System.getenv(Shell.WINDOWS ? "PATH"
- : "LD_LIBRARY_PATH");
- String hadoopHome = System.getenv("HADOOP_COMMON_HOME");
- if (hadoopHome == null) {
- hadoopHome = "";
- }
- String hadoopLibLocation = hadoopHome
- + (Shell.WINDOWS ? "\\bin" : "/lib/native");
- assertTrue(executionEnvPathVariable.contains(hadoopLibLocation));
- }
- }
// configure a job
private void configure(JobConf conf, Path inDir, Path outDir, String input,
@@ -167,57 +140,31 @@ public class TestMiniMRChildTask {
conf.set("test.build.data", TEST_ROOT_DIR);
}
- /**
- * Launch tests
- * @param conf Configuration of the mapreduce job.
- * @param inDir input path
- * @param outDir output path
- * @param input Input text
- * @throws IOException
- */
- public void launchTest(JobConf conf,
- Path inDir,
- Path outDir,
- String input)
- throws IOException, InterruptedException, ClassNotFoundException {
-
- FileSystem outFs = outDir.getFileSystem(conf);
-
- // Launch job with default option for temp dir.
- // i.e. temp dir is ./tmp
- Job job = Job.getInstance(conf);
- job.addFileToClassPath(APP_JAR);
- job.setJarByClass(TestMiniMRChildTask.class);
- job.setMaxMapAttempts(1); // speed up failures
- job.waitForCompletion(true);
- boolean succeeded = job.waitForCompletion(true);
- assertTrue(succeeded);
- outFs.delete(outDir, true);
- }
-
private static void checkEnv(String envName, String expValue, String mode) {
- String envValue = System.getenv(envName).trim();
+ String envValue = System.getenv(envName);
if ("append".equals(mode)) {
if (envValue == null || !envValue.contains(File.pathSeparator)) {
throw new RuntimeException("Missing env variable");
} else {
- String parts[] = envValue.split(File.pathSeparator);
+ String[] parts = envValue.trim().split(File.pathSeparator);
// check if the value is appended
if (!parts[parts.length - 1].equals(expValue)) {
throw new RuntimeException("Wrong env variable in append mode");
}
}
} else {
- if (envValue == null || !envValue.equals(expValue)) {
+ if (envValue == null || !envValue.trim().equals(expValue)) {
throw new RuntimeException("Wrong env variable in noappend mode");
}
}
}
// Mappers that simply checks if the desired user env are present or not
- static class EnvCheckMapper extends MapReduceBase implements
+ private static class EnvCheckMapper extends MapReduceBase implements
Mapper<WritableComparable, Writable, WritableComparable, Writable> {
-
+
+ @Override
+ @SuppressWarnings("deprecation")
public void configure(JobConf job) {
boolean oldConfigs = job.getBoolean(OLD_CONFIGS, false);
if (oldConfigs) {
@@ -236,38 +183,12 @@ public class TestMiniMRChildTask {
mapJavaOpts, MAP_OPTS_VAL);
}
- String path = job.get("path");
-
- // check if the pwd is there in LD_LIBRARY_PATH
- String pwd = System.getenv("PWD");
-
- assertTrue("LD doesnt contain pwd",
- System.getenv("LD_LIBRARY_PATH").contains(pwd));
-
- // check if X=$X:/abc works for LD_LIBRARY_PATH
- checkEnv("LD_LIBRARY_PATH", "/tmp", "append");
// check if X=y works for an already existing parameter
checkEnv("LANG", "en_us_8859_1", "noappend");
// check if X=/tmp for a new env variable
checkEnv("MY_PATH", "/tmp", "noappend");
// check if X=$X:/tmp works for a new env var and results into :/tmp
checkEnv("NEW_PATH", File.pathSeparator + "/tmp", "noappend");
- // check if X=$(tt's X var):/tmp for an old env variable inherited from
- // the tt
- if (Shell.WINDOWS) {
- // On Windows, PATH is replaced one more time as part of default config
- // of "mapreduce.admin.user.env", i.e. on Windows,
- // "mapreduce.admin.user.env" is set to
- // "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin"
- String hadoopHome = System.getenv("HADOOP_COMMON_HOME");
- if (hadoopHome == null) {
- hadoopHome = "";
- }
- String hadoopLibLocation = hadoopHome + "\\bin";
- path += File.pathSeparator + hadoopLibLocation;
- path += File.pathSeparator + path;
- }
- checkEnv("PATH", path + File.pathSeparator + "/tmp", "noappend");
String jobLocalDir = job.get(MRJobConfig.JOB_LOCAL_DIR);
assertNotNull(MRJobConfig.JOB_LOCAL_DIR + " is null",
@@ -281,10 +202,12 @@ public class TestMiniMRChildTask {
}
}
- static class EnvCheckReducer extends MapReduceBase
- implements Reducer<WritableComparable, Writable, WritableComparable, Writable> {
-
+ private static class EnvCheckReducer extends MapReduceBase
+ implements Reducer<WritableComparable, Writable,
+ WritableComparable, Writable> {
+
@Override
+ @SuppressWarnings("deprecation")
public void configure(JobConf job) {
boolean oldConfigs = job.getBoolean(OLD_CONFIGS, false);
if (oldConfigs) {
@@ -303,39 +226,12 @@ public class TestMiniMRChildTask {
reduceJavaOpts, REDUCE_OPTS_VAL);
}
- String path = job.get("path");
-
- // check if the pwd is there in LD_LIBRARY_PATH
- String pwd = System.getenv("PWD");
-
- assertTrue("LD doesnt contain pwd",
- System.getenv("LD_LIBRARY_PATH").contains(pwd));
-
- // check if X=$X:/abc works for LD_LIBRARY_PATH
- checkEnv("LD_LIBRARY_PATH", "/tmp", "append");
// check if X=y works for an already existing parameter
checkEnv("LANG", "en_us_8859_1", "noappend");
// check if X=/tmp for a new env variable
checkEnv("MY_PATH", "/tmp", "noappend");
// check if X=$X:/tmp works for a new env var and results into :/tmp
checkEnv("NEW_PATH", File.pathSeparator + "/tmp", "noappend");
- // check if X=$(tt's X var):/tmp for an old env variable inherited from
- // the tt
- if (Shell.WINDOWS) {
- // On Windows, PATH is replaced one more time as part of default config
- // of "mapreduce.admin.user.env", i.e. on Windows,
- // "mapreduce.admin.user.env"
- // is set to "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin"
- String hadoopHome = System.getenv("HADOOP_COMMON_HOME");
- if (hadoopHome == null) {
- hadoopHome = "";
- }
- String hadoopLibLocation = hadoopHome + "\\bin";
- path += File.pathSeparator + hadoopLibLocation;
- path += File.pathSeparator + path;
- }
- checkEnv("PATH", path + File.pathSeparator + "/tmp", "noappend");
-
}
@Override
@@ -393,72 +289,9 @@ public class TestMiniMRChildTask {
}
/**
- * To test OS dependent setting of default execution path for a MapRed task.
- * Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -
- * for WINDOWS: %HADOOP_COMMON_HOME%\bin is expected to be included in PATH -
- * for Linux: $HADOOP_COMMON_HOME/lib/native is expected to be included in
- * LD_LIBRARY_PATH
- */
- @Test
- public void testMapRedExecutionEnv() {
- // for windows, test if the env variable can be set
- // this may be removed as part of MAPREDUCE-6588
- if (Shell.WINDOWS) {
- try {
- // Application environment
- Map<String, String> environment = new HashMap<String, String>();
- String setupHadoopHomeCommand =
- "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home";
- MRApps.setEnvFromInputString(environment, setupHadoopHomeCommand, conf);
-
- // Add the env variables passed by the admin
- MRApps.setEnvFromInputString(environment, conf.get(
- MRJobConfig.MAPRED_ADMIN_USER_ENV,
- MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
-
- String executionPaths = environment.get("PATH");
- String toFind =
- "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin";
-
- // Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
- assertTrue("execution path does not include the hadoop lib location "
- + toFind, executionPaths.contains(toFind));
- } catch (Exception e) {
- e.printStackTrace();
- fail("Exception in testing execution environment for MapReduce task");
- tearDown();
- }
- }
-
- // now launch a mapreduce job to ensure that the child
- // also gets the configured setting for hadoop lib
- try {
-
- JobConf conf = new JobConf(mr.getConfig());
- // initialize input, output directories
- Path rootDir = new Path(System.getProperty("test.build.data",
- "build/test/data"));
- Path inDir = new Path(rootDir, "input");
- Path outDir = new Path(rootDir, "output");
- String input = "The input";
-
- // set config to use the ExecutionEnvCheckMapClass map class
- configure(conf, inDir, outDir, input,
- ExecutionEnvCheckMapClass.class, IdentityReducer.class);
- launchTest(conf, inDir, outDir, input);
-
- } catch(Exception e) {
- e.printStackTrace();
- fail("Exception in testing propagation of env setting to child task");
- tearDown();
- }
- }
-
- /**
* Test to test if the user set env variables reflect in the child
* processes. Mainly
* - x=y (x can be a already existing env variable or a new variable)
- * - x=$x:y (replace $x with the current value of x)
*/
@Test
public void testTaskEnv(){
@@ -482,10 +315,9 @@ public class TestMiniMRChildTask {
* Test to test if the user set *old* env variables reflect in the child
* processes. Mainly
* - x=y (x can be a already existing env variable or a new variable)
- * - x=$x:y (replace $x with the current value of x)
*/
@Test
- public void testTaskOldEnv(){
+ public void testTaskOldEnv() {
try {
JobConf conf = new JobConf(mr.getConfig());
String baseDir = System.getProperty("test.build.data", "build/test/data");
@@ -495,23 +327,23 @@ public class TestMiniMRChildTask {
FileSystem outFs = outDir.getFileSystem(conf);
runTestTaskEnv(conf, inDir, outDir, true);
outFs.delete(outDir, true);
- } catch(Exception e) {
+ } catch (Exception e) {
e.printStackTrace();
fail("Exception in testing child env");
tearDown();
}
}
-
- void runTestTaskEnv(JobConf conf, Path inDir, Path outDir, boolean oldConfigs)
- throws IOException, InterruptedException, ClassNotFoundException {
+
+ @SuppressWarnings("deprecation")
+ private void runTestTaskEnv(JobConf config, Path inDir, Path outDir,
+ boolean oldConfigs)
+ throws IOException, InterruptedException, ClassNotFoundException {
String input = "The input";
- configure(conf, inDir, outDir, input,
+ configure(config, inDir, outDir, input,
EnvCheckMapper.class, EnvCheckReducer.class);
// test
// - new SET of new var (MY_PATH)
// - set of old var (LANG)
- // - append to an old var from modified env (LD_LIBRARY_PATH)
- // - append to an old var from tt's env (PATH)
// - append to a new var (NEW_PATH)
String mapTaskEnvKey = JobConf.MAPRED_MAP_TASK_ENV;
String reduceTaskEnvKey = JobConf.MAPRED_MAP_TASK_ENV;
@@ -519,29 +351,24 @@ public class TestMiniMRChildTask {
String reduceTaskJavaOptsKey = JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS;
String mapTaskJavaOpts = MAP_OPTS_VAL;
String reduceTaskJavaOpts = REDUCE_OPTS_VAL;
- conf.setBoolean(OLD_CONFIGS, oldConfigs);
+ config.setBoolean(OLD_CONFIGS, oldConfigs);
if (oldConfigs) {
mapTaskEnvKey = reduceTaskEnvKey = JobConf.MAPRED_TASK_ENV;
mapTaskJavaOptsKey = reduceTaskJavaOptsKey = JobConf.MAPRED_TASK_JAVA_OPTS;
mapTaskJavaOpts = reduceTaskJavaOpts = TASK_OPTS_VAL;
}
- conf.set(
+ config.set(
mapTaskEnvKey,
- Shell.WINDOWS ? "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=%LD_LIBRARY_PATH%;/tmp,"
- + "PATH=%PATH%;/tmp,NEW_PATH=%NEW_PATH%;/tmp"
- : "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
- + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
- conf.set(
+ Shell.WINDOWS ? "MY_PATH=/tmp,LANG=en_us_8859_1,NEW_PATH=%MY_PATH%;/tmp"
+ : "MY_PATH=/tmp,LANG=en_us_8859_1,NEW_PATH=$NEW_PATH:/tmp");
+ config.set(
reduceTaskEnvKey,
- Shell.WINDOWS ? "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=%LD_LIBRARY_PATH%;/tmp,"
- + "PATH=%PATH%;/tmp,NEW_PATH=%NEW_PATH%;/tmp"
- : "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp,"
- + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp");
- conf.set("path", System.getenv("PATH"));
- conf.set(mapTaskJavaOptsKey, mapTaskJavaOpts);
- conf.set(reduceTaskJavaOptsKey, reduceTaskJavaOpts);
+ Shell.WINDOWS ? "MY_PATH=/tmp,LANG=en_us_8859_1,NEW_PATH=%MY_PATH%;/tmp"
+ : "MY_PATH=/tmp,LANG=en_us_8859_1,NEW_PATH=$NEW_PATH:/tmp");
+ config.set(mapTaskJavaOptsKey, mapTaskJavaOpts);
+ config.set(reduceTaskJavaOptsKey, reduceTaskJavaOpts);
- Job job = Job.getInstance(conf);
+ Job job = Job.getInstance(config);
job.addFileToClassPath(APP_JAR);
job.setJarByClass(TestMiniMRChildTask.class);
job.setMaxMapAttempts(1); // speed up failures
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org