You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by dd...@apache.org on 2011/06/11 03:39:35 UTC
svn commit: r1134497 - in /hadoop/mapreduce/branches/MR-279: CHANGES.txt
mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Author: ddas
Date: Sat Jun 11 01:39:35 2011
New Revision: 1134497
URL: http://svn.apache.org/viewvc?rev=1134497&view=rev
Log:
Changes a couple of usages of FileContext to FileSystem in YarnRunner to handle distributed cache path resolutions on non-default filesystems. Contributed by Devaraj Das.
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1134497&r1=1134496&r2=1134497&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Sat Jun 11 01:39:35 2011
@@ -5,6 +5,10 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Changes a couple of usages of FileContext to FileSystem in YarnRunner
+ to handle distributed cache path resolutions on non-default filesystems.
+ (ddas)
+
Fix to exclude images dir into the tar distribution (luke lu via gkesavan)
Fix more rounding off problems in reduce ramp up. Also fix a bug preventing
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1134497&r1=1134496&r2=1134497&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Sat Jun 11 01:39:35 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -380,7 +381,7 @@ public class YARNRunner implements Clien
ApplicationSubmissionContext container) throws IOException {
// Cache archives
- parseDistributedCacheArtifacts(container, LocalResourceType.ARCHIVE,
+ parseDistributedCacheArtifacts(conf, container, LocalResourceType.ARCHIVE,
DistributedCache.getCacheArchives(conf),
DistributedCache.getArchiveTimestamps(conf),
getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES),
@@ -388,7 +389,7 @@ public class YARNRunner implements Clien
DistributedCache.getArchiveClassPaths(conf));
// Cache files
- parseDistributedCacheArtifacts(container, LocalResourceType.FILE,
+ parseDistributedCacheArtifacts(conf, container, LocalResourceType.FILE,
DistributedCache.getCacheFiles(conf),
DistributedCache.getFileTimestamps(conf),
getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
@@ -398,7 +399,7 @@ public class YARNRunner implements Clien
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], long[], boolean[], Path[], FileType)
- private void parseDistributedCacheArtifacts(
+ private void parseDistributedCacheArtifacts(Configuration conf,
ApplicationSubmissionContext container, LocalResourceType type,
URI[] uris, long[] timestamps, long[] sizes, boolean visibilities[],
Path[] pathsToPutOnClasspath) throws IOException {
@@ -418,17 +419,17 @@ public class YARNRunner implements Clien
Map<String, Path> classPaths = new HashMap<String, Path>();
if (pathsToPutOnClasspath != null) {
for (Path p : pathsToPutOnClasspath) {
- p = p.makeQualified(this.defaultFileContext.getDefaultFileSystem()
- .getUri(), this.defaultFileContext.getWorkingDirectory());
+ FileSystem fs = p.getFileSystem(conf);
+ p = p.makeQualified(fs.getUri(), fs.getWorkingDirectory());
classPaths.put(p.toUri().getPath().toString(), p);
}
}
for (int i = 0; i < uris.length; ++i) {
URI u = uris[i];
Path p = new Path(u);
- p = defaultFileContext.getDefaultFileSystem().resolvePath(
- p.makeQualified(this.defaultFileContext.getDefaultFileSystem()
- .getUri(), this.defaultFileContext.getWorkingDirectory()));
+ FileSystem fs = p.getFileSystem(conf);
+ p = fs.resolvePath(
+ p.makeQualified(fs.getUri(), fs.getWorkingDirectory()));
// Add URI fragment or just the filename
Path name = new Path((null == u.getFragment())
? p.getName()