You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/08/12 16:38:05 UTC
[20/50] [abbrv] hadoop git commit: HADOOP-10392. Use
FileSystem#makeQualified(Path) instead of Path#makeQualified(FileSystem)
(ajisakaa via aw)
HADOOP-10392. Use FileSystem#makeQualified(Path) instead of Path#makeQualified(FileSystem) (ajisakaa via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4222c971
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4222c971
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4222c971
Branch: refs/heads/HDFS-10467
Commit: 4222c971080f2b150713727092c7197df58c88e5
Parents: d964062
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Aug 11 09:25:56 2017 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Aug 11 09:25:56 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileUtil.java | 4 +--
.../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 4 +--
.../java/org/apache/hadoop/io/SequenceFile.java | 2 +-
.../apache/hadoop/fs/TestLocalFileSystem.java | 6 ++---
.../java/org/apache/hadoop/io/FileBench.java | 2 +-
.../mapred/MiniMRClientClusterFactory.java | 4 +--
.../mapred/TestCombineFileInputFormat.java | 6 ++---
.../TestCombineSequenceFileInputFormat.java | 7 +++--
.../mapred/TestCombineTextInputFormat.java | 7 +++--
.../mapred/TestConcatenatedCompressedInput.java | 6 ++---
.../org/apache/hadoop/mapred/TestMapRed.java | 4 +--
.../hadoop/mapred/TestMiniMRChildTask.java | 4 +--
.../hadoop/mapred/TestTextInputFormat.java | 8 +++---
.../TestWrappedRecordReaderClassloader.java | 4 +--
.../lib/join/TestWrappedRRClassloader.java | 4 +--
.../mapreduce/util/MRAsyncDiskService.java | 2 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 4 +--
.../v2/TestMRJobsWithHistoryService.java | 4 +--
.../org/apache/hadoop/tools/HadoopArchives.java | 2 +-
.../apache/hadoop/mapred/gridmix/Gridmix.java | 2 +-
.../hadoop/mapred/gridmix/PseudoLocalFs.java | 8 +++++-
.../hadoop/mapred/gridmix/TestFilePool.java | 4 +--
.../hadoop/mapred/gridmix/TestFileQueue.java | 8 +++---
.../mapred/gridmix/TestPseudoLocalFs.java | 2 +-
.../hadoop/mapred/gridmix/TestUserResolve.java | 4 +--
.../hadoop/fs/swift/util/SwiftTestUtils.java | 2 +-
.../fs/swift/SwiftFileSystemBaseTest.java | 2 +-
.../TestSwiftFileSystemPartitionedUploads.java | 4 +--
.../hadoop/tools/rumen/TestHistograms.java | 6 ++---
.../org/apache/hadoop/streaming/StreamJob.java | 27 ++++++++++----------
30 files changed, 78 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index eb8a5c3..72b9615 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -295,8 +295,8 @@ public class FileUtil {
Path dst)
throws IOException {
if (srcFS == dstFS) {
- String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
- String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
+ String srcq = srcFS.makeQualified(src).toString() + Path.SEPARATOR;
+ String dstq = dstFS.makeQualified(dst).toString() + Path.SEPARATOR;
if (dstq.startsWith(srcq)) {
if (srcq.length() == dstq.length()) {
throw new IOException("Cannot copy " + src + " to itself.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index 4c1236b..644cf4e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -505,7 +505,7 @@ public class FTPFileSystem extends FileSystem {
long modTime = -1; // Modification time of root dir not known.
Path root = new Path("/");
return new FileStatus(length, isDir, blockReplication, blockSize,
- modTime, root.makeQualified(this));
+ modTime, this.makeQualified(root));
}
String pathName = parentPath.toUri().getPath();
FTPFile[] ftpFiles = client.listFiles(pathName);
@@ -546,7 +546,7 @@ public class FTPFileSystem extends FileSystem {
String group = ftpFile.getGroup();
Path filePath = new Path(parentPath, ftpFile.getName());
return new FileStatus(length, isDir, blockReplication, blockSize, modTime,
- accessTime, permission, user, group, filePath.makeQualified(this));
+ accessTime, permission, user, group, this.makeQualified(filePath));
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index 2cc0e40..f42848b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -1883,7 +1883,7 @@ public class SequenceFile {
@Deprecated
public Reader(FileSystem fs, Path file,
Configuration conf) throws IOException {
- this(conf, file(file.makeQualified(fs)));
+ this(conf, file(fs.makeQualified(file)));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
index 357c683..90eaa2a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
@@ -218,8 +218,8 @@ public class TestLocalFileSystem {
@Test
public void testHomeDirectory() throws IOException {
- Path home = new Path(System.getProperty("user.home"))
- .makeQualified(fileSys);
+ Path home = fileSys.makeQualified(
+ new Path(System.getProperty("user.home")));
Path fsHome = fileSys.getHomeDirectory();
assertEquals(home, fsHome);
}
@@ -229,7 +229,7 @@ public class TestLocalFileSystem {
Path path = new Path(TEST_ROOT_DIR, "foo%bar");
writeFile(fileSys, path, 1);
FileStatus status = fileSys.getFileStatus(path);
- assertEquals(path.makeQualified(fileSys), status.getPath());
+ assertEquals(fileSys.makeQualified(path), status.getPath());
cleanupFile(fileSys, path);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java
index 0a9d0e9..ef68cdf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java
@@ -170,7 +170,7 @@ public class FileBench extends Configured implements Tool {
for(int i = 0; i < argv.length; ++i) {
try {
if ("-dir".equals(argv[i])) {
- root = new Path(argv[++i]).makeQualified(fs);
+ root = fs.makeQualified(new Path(argv[++i]));
System.out.println("DIR: " + root.toString());
} else if ("-seed".equals(argv[i])) {
job.setLong("filebench.seed", Long.valueOf(argv[++i]));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java
index 023da48..85c534b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java
@@ -50,8 +50,8 @@ public class MiniMRClientClusterFactory {
FileSystem fs = FileSystem.get(conf);
- Path testRootDir = new Path("target", identifier + "-tmpDir")
- .makeQualified(fs);
+ Path testRootDir = fs.makeQualified(
+ new Path("target", identifier + "-tmpDir"));
Path appJar = new Path(testRootDir, "MRAppJar.jar");
// Copy MRAppJar and make it private.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
index ca3c2df..de7880d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
@@ -47,9 +47,9 @@ public class TestCombineFileInputFormat {
throw new RuntimeException("init failure", e);
}
}
- private static Path workDir =
- new Path(new Path(System.getProperty("test.build.data", "/tmp")),
- "TestCombineFileInputFormat").makeQualified(localFs);
+ private static Path workDir = localFs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"),
+ "TestCombineFileInputFormat"));
private static void writeFile(FileSystem fs, Path name,
String contents) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java
index 8d0203e..8cdaa80 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java
@@ -53,10 +53,9 @@ public class TestCombineSequenceFileInputFormat {
}
}
- @SuppressWarnings("deprecation")
- private static Path workDir =
- new Path(new Path(System.getProperty("test.build.data", "/tmp")),
- "TestCombineSequenceFileInputFormat").makeQualified(localFs);
+ private static Path workDir = localFs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"),
+ "TestCombineSequenceFileInputFormat"));
@Test(timeout=10000)
public void testFormat() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java
index ca86dd5..581e62b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java
@@ -60,10 +60,9 @@ public class TestCombineTextInputFormat {
}
}
- @SuppressWarnings("deprecation")
- private static Path workDir =
- new Path(new Path(System.getProperty("test.build.data", "/tmp")),
- "TestCombineTextInputFormat").makeQualified(localFs);
+ private static Path workDir = localFs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"),
+ "TestCombineTextInputFormat"));
// A reporter that does nothing
private static final Reporter voidReporter = Reporter.NULL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java
index 22a05c5..15d651d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java
@@ -84,9 +84,9 @@ public class TestConcatenatedCompressedInput {
public void after() {
ZlibFactory.loadNativeZLib();
}
- private static Path workDir =
- new Path(new Path(System.getProperty("test.build.data", "/tmp")),
- "TestConcatenatedCompressedInput").makeQualified(localFs);
+ private static Path workDir = localFs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"),
+ "TestConcatenatedCompressedInput"));
private static LineReader makeStream(String str) throws IOException {
return new LineReader(new ByteArrayInputStream(str.getBytes("UTF-8")),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java
index d60905e..af09e09 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java
@@ -342,8 +342,8 @@ public class TestMapRed extends Configured implements Tool {
values.add(m);
m = m.replace((char)('A' + i - 1), (char)('A' + i));
}
- Path testdir = new Path(
- System.getProperty("test.build.data","/tmp")).makeQualified(fs);
+ Path testdir = fs.makeQualified(new Path(
+ System.getProperty("test.build.data","/tmp")));
fs.delete(testdir, true);
Path inFile = new Path(testdir, "nullin/blah");
SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, inFile,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
index f690118..51f0120 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
@@ -75,8 +75,8 @@ public class TestMiniMRChildTask {
}
}
- private static Path TEST_ROOT_DIR = new Path("target",
- TestMiniMRChildTask.class.getName() + "-tmpDir").makeQualified(localFs);
+ private static Path TEST_ROOT_DIR = localFs.makeQualified(
+ new Path("target", TestMiniMRChildTask.class.getName() + "-tmpDir"));
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
index 5106c38..67bd497 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
@@ -61,10 +61,10 @@ public class TestTextInputFormat {
throw new RuntimeException("init failure", e);
}
}
- @SuppressWarnings("deprecation")
- private static Path workDir =
- new Path(new Path(System.getProperty("test.build.data", "/tmp")),
- "TestTextInputFormat").makeQualified(localFs);
+
+ private static Path workDir = localFs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"),
+ "TestTextInputFormat"));
@Test (timeout=500000)
public void testFormat() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java
index ae5572f..785898d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java
@@ -50,8 +50,8 @@ public class TestWrappedRecordReaderClassloader {
assertTrue(job.getClassLoader() instanceof Fake_ClassLoader);
FileSystem fs = FileSystem.get(job);
- Path testdir = new Path(System.getProperty("test.build.data", "/tmp"))
- .makeQualified(fs);
+ Path testdir = fs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp")));
Path base = new Path(testdir, "/empty");
Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java
index 680e246..e3d7fa0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java
@@ -50,8 +50,8 @@ public class TestWrappedRRClassloader {
assertTrue(conf.getClassLoader() instanceof Fake_ClassLoader);
FileSystem fs = FileSystem.get(conf);
- Path testdir = new Path(System.getProperty("test.build.data", "/tmp"))
- .makeQualified(fs);
+ Path testdir = fs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp")));
Path base = new Path(testdir, "/empty");
Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java
index 4446756..be46385 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java
@@ -330,7 +330,7 @@ public class MRAsyncDiskService {
* Returns the normalized path of a path.
*/
private String normalizePath(String path) {
- return (new Path(path)).makeQualified(this.localFileSystem)
+ return this.localFileSystem.makeQualified(new Path(path))
.toUri().getPath();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index c6d2168..274f405 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -128,8 +128,8 @@ public class TestMRJobs {
}
}
- private static Path TEST_ROOT_DIR = new Path("target",
- TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs);
+ private static Path TEST_ROOT_DIR = localFs.makeQualified(
+ new Path("target", TestMRJobs.class.getName() + "-tmpDir"));
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
private static final String OUTPUT_ROOT_DIR = "/tmp/" +
TestMRJobs.class.getSimpleName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
index f9236a9..98a6de2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
@@ -73,8 +73,8 @@ public class TestMRJobsWithHistoryService {
}
}
- private static Path TEST_ROOT_DIR = new Path("target",
- TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs);
+ private static Path TEST_ROOT_DIR = localFs.makeQualified(
+ new Path("target", TestMRJobs.class.getName() + "-tmpDir"));
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
@Before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
index c2097dc..8ad8600 100644
--- a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
+++ b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
@@ -473,7 +473,7 @@ public class HadoopArchives implements Tool {
conf.setLong(HAR_BLOCKSIZE_LABEL, blockSize);
conf.setLong(HAR_PARTSIZE_LABEL, partSize);
conf.set(DST_HAR_LABEL, archiveName);
- conf.set(SRC_PARENT_LABEL, parentPath.makeQualified(fs).toString());
+ conf.set(SRC_PARENT_LABEL, fs.makeQualified(parentPath).toString());
conf.setInt(HAR_REPLICATION_LABEL, repl);
Path outputPath = new Path(dest, archiveName);
FileOutputFormat.setOutputPath(conf, outputPath);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
index 4386bc1..3507b7f 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
@@ -447,7 +447,7 @@ public class Gridmix extends Configured implements Tool {
// Create <ioPath> with 777 permissions
final FileSystem inputFs = ioPath.getFileSystem(conf);
- ioPath = ioPath.makeQualified(inputFs);
+ ioPath = inputFs.makeQualified(ioPath);
boolean succeeded = false;
try {
succeeded = FileSystem.mkdirs(inputFs, ioPath,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
index d7ef563..15fc68e 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java
@@ -116,7 +116,7 @@ class PseudoLocalFs extends FileSystem {
* @throws FileNotFoundException
*/
long validateFileNameFormat(Path path) throws FileNotFoundException {
- path = path.makeQualified(this);
+ path = this.makeQualified(path);
boolean valid = true;
long fileSize = 0;
if (!path.toUri().getScheme().equals(getUri().getScheme())) {
@@ -329,4 +329,10 @@ class PseudoLocalFs extends FileSystem {
throw new UnsupportedOperationException("SetWorkingDirectory "
+ "is not supported in pseudo local file system.");
}
+
+ @Override
+ public Path makeQualified(Path path) {
+ // skip FileSystem#checkPath() to validate some other Filesystems
+ return path.makeQualified(this.getUri(), this.getWorkingDirectory());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java
index 4be90c6..a75414a 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java
@@ -48,8 +48,8 @@ public class TestFilePool {
try {
final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
- return new Path(System.getProperty("test.build.data", "/tmp"),
- "testFilePool").makeQualified(fs);
+ return fs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"), "testFilePool"));
} catch (IOException e) {
fail();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java
index a4668ee..e68e83f 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java
@@ -48,8 +48,8 @@ public class TestFileQueue {
public static void setup() throws IOException {
final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
- final Path p = new Path(System.getProperty("test.build.data", "/tmp"),
- "testFileQueue").makeQualified(fs);
+ final Path p = fs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"), "testFileQueue"));
fs.delete(p, true);
final byte[] b = new byte[BLOCK];
for (int i = 0; i < NFILES; ++i) {
@@ -71,8 +71,8 @@ public class TestFileQueue {
public static void cleanup() throws IOException {
final Configuration conf = new Configuration();
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
- final Path p = new Path(System.getProperty("test.build.data", "/tmp"),
- "testFileQueue").makeQualified(fs);
+ final Path p = fs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"), "testFileQueue"));
fs.delete(p, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java
index a607ece..7179c5d 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java
@@ -224,7 +224,7 @@ public class TestPseudoLocalFs {
// Validate operations on valid qualified path
path = new Path("myPsedoFile.1237");
- path = path.makeQualified(pfs);
+ path = pfs.makeQualified(path);
validateGetFileStatus(pfs, path, true);
validateCreate(pfs, path, true);
validateOpen(pfs, path, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
index 8050f33..4407515 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java
@@ -40,8 +40,8 @@ public class TestUserResolve {
public static void createRootDir() throws IOException {
conf = new Configuration();
fs = FileSystem.getLocal(conf);
- rootDir = new Path(new Path(System.getProperty("test.build.data", "/tmp"))
- .makeQualified(fs), "gridmixUserResolve");
+ rootDir = new Path(fs.makeQualified(new Path(
+ System.getProperty("test.build.data", "/tmp"))), "gridmixUserResolve");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
index f91ba30..726045e 100644
--- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
+++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java
@@ -278,7 +278,7 @@ public class SwiftTestUtils extends org.junit.Assert {
noteAction(action);
try {
if (fileSystem != null) {
- fileSystem.delete(new Path(cleanupPath).makeQualified(fileSystem),
+ fileSystem.delete(fileSystem.makeQualified(new Path(cleanupPath)),
true);
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java
index 12f58e6..99e03c7 100644
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java
@@ -159,7 +159,7 @@ public class SwiftFileSystemBaseTest extends Assert implements
* @return a qualified path instance
*/
protected Path path(String pathString) {
- return new Path(pathString).makeQualified(fs);
+ return fs.makeQualified(new Path(pathString));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
index f344093..b42abcd 100644
--- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
+++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java
@@ -126,7 +126,7 @@ public class TestSwiftFileSystemPartitionedUploads extends
SwiftTestUtils.compareByteArrays(src, dest, len);
FileStatus status;
- final Path qualifiedPath = path.makeQualified(fs);
+ final Path qualifiedPath = fs.makeQualified(path);
status = fs.getFileStatus(qualifiedPath);
//now see what block location info comes back.
//This will vary depending on the Swift version, so the results
@@ -216,7 +216,7 @@ public class TestSwiftFileSystemPartitionedUploads extends
private FileStatus validatePathLen(Path path, int len) throws IOException {
//verify that the length is what was written in a direct status check
- final Path qualifiedPath = path.makeQualified(fs);
+ final Path qualifiedPath = fs.makeQualified(path);
FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
StringBuilder listing = lsToString(parentDirListing);
String parentDirLS = listing.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
index 206095a..52caaf5 100644
--- a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
+++ b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
@@ -57,8 +57,8 @@ public class TestHistograms {
public void testHistograms() throws IOException {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
- final Path rootInputDir = new Path(
- System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
+ final Path rootInputDir = lfs.makeQualified(new Path(
+ System.getProperty("test.tools.input.dir", "target/input")));
final Path rootInputFile = new Path(rootInputDir, "rumen/histogram-tests");
@@ -132,7 +132,7 @@ public class TestHistograms {
final FileSystem lfs = FileSystem.getLocal(conf);
for (String arg : args) {
- Path filePath = new Path(arg).makeQualified(lfs);
+ Path filePath = lfs.makeQualified(new Path(arg));
String fileName = filePath.getName();
if (fileName.startsWith("input")) {
LoggedDiscreteCDF newResult = histogramFileToCDF(filePath, lfs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4222c971/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
index 9f5b293..0b239d0 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
@@ -22,13 +22,11 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
-import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.regex.Pattern;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -41,12 +39,12 @@ import org.apache.commons.cli.Options;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.mapred.FileInputFormat;
@@ -56,7 +54,6 @@ import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
-import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.SequenceFileAsTextInputFormat;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
@@ -65,6 +62,7 @@ import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.LazyOutputFormat;
import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner;
import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.streaming.io.IdentifierResolver;
import org.apache.hadoop.streaming.io.InputWriter;
import org.apache.hadoop.streaming.io.OutputReader;
@@ -297,7 +295,10 @@ public class StreamJob implements Tool {
try {
Path path = new Path(file);
FileSystem localFs = FileSystem.getLocal(config_);
- String finalPath = path.makeQualified(localFs).toString();
+ Path qualifiedPath = path.makeQualified(
+ localFs.getUri(), localFs.getWorkingDirectory());
+ validate(qualifiedPath);
+ String finalPath = qualifiedPath.toString();
if(fileList.length() > 0) {
fileList.append(',');
}
@@ -313,7 +314,6 @@ public class StreamJob implements Tool {
tmpFiles = tmpFiles + "," + fileList;
}
config_.set("tmpfiles", tmpFiles);
- validate(packageFiles_);
}
String fsName = cmdLine.getOptionValue("dfs");
@@ -391,14 +391,13 @@ public class StreamJob implements Tool {
return OptionBuilder.withDescription(desc).create(name);
}
- private void validate(final List<String> values)
- throws IllegalArgumentException {
- for (String file : values) {
- File f = new File(file);
- if (!FileUtil.canRead(f)) {
- fail("File: " + f.getAbsolutePath()
- + " does not exist, or is not readable.");
- }
+ private void validate(final Path path) throws IOException {
+ try {
+ path.getFileSystem(config_).access(path, FsAction.READ);
+ } catch (FileNotFoundException e) {
+ fail("File: " + path + " does not exist.");
+ } catch (AccessControlException e) {
+ fail("File: " + path + " is not readable.");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org