You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cn...@apache.org on 2013/07/03 01:39:34 UTC
svn commit: r1499151 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project: CHANGES.txt
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
Author: cnauroth
Date: Tue Jul 2 23:39:34 2013
New Revision: 1499151
URL: http://svn.apache.org/r1499151
Log:
MAPREDUCE-5355. Merging change r1499149 from branch-2 to branch-2.1-beta.
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt?rev=1499151&r1=1499150&r2=1499151&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt Tue Jul 2 23:39:34 2013
@@ -436,6 +436,9 @@ Release 2.1.0-beta - 2013-07-02
MAPREDUCE-5349. TestClusterMapReduceTestCase and TestJobName fail on Windows
in branch-2. (Chuan Liu via cnauroth)
+ MAPREDUCE-5355. MiniMRYarnCluster with localFs does not work on Windows.
+ (Chuan Liu via cnauroth)
+
MAPREDUCE-5291. Change MR App to use updated property names in
container-log4j.properties. (Zhijie Shen via sseth)
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1499151&r1=1499150&r2=1499151&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Tue Jul 2 23:39:34 2013
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.LocalContainerLauncher;
import org.apache.hadoop.mapred.ShuffleHandler;
@@ -87,6 +88,21 @@ public class MiniMRYarnCluster extends M
try {
Path stagingPath = FileContext.getFileContext(conf).makeQualified(
new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
+ /*
+ * Re-configure the staging path on Windows if the file system is localFs.
+ * We need to use a absolute path that contains the drive letter. The unit
+ * test could run on a different drive than the AM. We can run into the
+ * issue that job files are localized to the drive where the test runs on,
+ * while the AM starts on a different drive and fails to find the job
+ * metafiles. Using absolute path can avoid this ambiguity.
+ */
+ if (Path.WINDOWS) {
+ if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
+ conf.set(MRJobConfig.MR_AM_STAGING_DIR,
+ new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR))
+ .getAbsolutePath());
+ }
+ }
FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
if (fc.util().exists(stagingPath)) {
LOG.info(stagingPath + " exists! deleting...");