You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by bo...@apache.org on 2012/11/05 16:54:38 UTC
svn commit: r1405851 - in
/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src:
main/java/org/apache/hadoop/yarn/util/FSDownload.java
test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
Author: bobby
Date: Mon Nov 5 15:54:38 2012
New Revision: 1405851
URL: http://svn.apache.org/viewvc?rev=1405851&view=rev
Log:
svn merge -c 1405846. Localization Update
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1405851&r1=1405850&r2=1405851&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java Mon Nov 5 15:54:38 2012
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.RunJar;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -90,6 +91,85 @@ public class FSDownload implements Calla
}
}
+ /**
+ * Returns a boolean to denote whether a cache file is visible to all(public)
+ * or not
+ * @param conf
+ * @param uri
+ * @return true if the path in the uri is visible to all, false otherwise
+ * @throws IOException
+ */
+ private static boolean isPublic(FileSystem fs, Path current) throws IOException {
+ current = fs.makeQualified(current);
+ //the leaf level file should be readable by others
+ if (!checkPublicPermsForAll(fs, current, FsAction.READ_EXECUTE, FsAction.READ)) {
+ return false;
+ }
+ return ancestorsHaveExecutePermissions(fs, current.getParent());
+ }
+
+ private static boolean checkPublicPermsForAll(FileSystem fs, Path current,
+ FsAction dir, FsAction file)
+ throws IOException {
+ return checkPublicPermsForAll(fs, fs.getFileStatus(current), dir, file);
+ }
+
+ private static boolean checkPublicPermsForAll(FileSystem fs,
+ FileStatus status, FsAction dir, FsAction file)
+ throws IOException {
+ FsPermission perms = status.getPermission();
+ FsAction otherAction = perms.getOtherAction();
+ if (status.isDirectory()) {
+ if (!otherAction.implies(dir)) {
+ return false;
+ }
+
+ for (FileStatus child : fs.listStatus(status.getPath())) {
+ if(!checkPublicPermsForAll(fs, child, dir, file)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return (otherAction.implies(file));
+ }
+
+ /**
+ * Returns true if all ancestors of the specified path have the 'execute'
+ * permission set for all users (i.e. that other users can traverse
+ * the directory heirarchy to the given path)
+ */
+ private static boolean ancestorsHaveExecutePermissions(FileSystem fs, Path path)
+ throws IOException {
+ Path current = path;
+ while (current != null) {
+ //the subdirs in the path should have execute permissions for others
+ if (!checkPermissionOfOther(fs, current, FsAction.EXECUTE)) {
+ return false;
+ }
+ current = current.getParent();
+ }
+ return true;
+ }
+
+ /**
+ * Checks for a given path whether the Other permissions on it
+ * imply the permission in the passed FsAction
+ * @param fs
+ * @param path
+ * @param action
+ * @return true if the path in the uri is visible to all, false otherwise
+ * @throws IOException
+ */
+ private static boolean checkPermissionOfOther(FileSystem fs, Path path,
+ FsAction action) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ FsPermission perms = status.getPermission();
+ FsAction otherAction = perms.getOtherAction();
+ return otherAction.implies(action);
+ }
+
+
private Path copy(Path sCopy, Path dstdir) throws IOException {
FileSystem sourceFs = sCopy.getFileSystem(conf);
Path dCopy = new Path(dstdir, sCopy.getName() + ".tmp");
@@ -99,7 +179,14 @@ public class FSDownload implements Calla
" changed on src filesystem (expected " + resource.getTimestamp() +
", was " + sStat.getModificationTime());
}
-
+ if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
+ if (!isPublic(sourceFs, sCopy)) {
+ throw new IOException("Resource " + sCopy +
+ " is not publicly accessable and as such cannot be part of the" +
+ " public cache.");
+ }
+ }
+
sourceFs.copyToLocalFile(sCopy, dCopy);
return dCopy;
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java?rev=1405851&r1=1405850&r2=1405851&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java Mon Nov 5 15:54:38 2012
@@ -114,6 +114,54 @@ public class TestFSDownload {
}
@Test
+ public void testDownloadBadPublic() throws IOException, URISyntaxException,
+ InterruptedException {
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+ FileContext files = FileContext.getLocalFSFileContext(conf);
+ final Path basedir = files.makeQualified(new Path("target",
+ TestFSDownload.class.getSimpleName()));
+ files.mkdir(basedir, null, true);
+ conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
+
+ Map<LocalResource, LocalResourceVisibility> rsrcVis =
+ new HashMap<LocalResource, LocalResourceVisibility>();
+
+ Random rand = new Random();
+ long sharedSeed = rand.nextLong();
+ rand.setSeed(sharedSeed);
+ System.out.println("SEED: " + sharedSeed);
+
+ Map<LocalResource,Future<Path>> pending =
+ new HashMap<LocalResource,Future<Path>>();
+ ExecutorService exec = Executors.newSingleThreadExecutor();
+ LocalDirAllocator dirs =
+ new LocalDirAllocator(TestFSDownload.class.getName());
+ int size = 512;
+ LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
+ Path path = new Path(basedir, "test-file");
+ LocalResource rsrc = createFile(files, path, size, rand, vis);
+ rsrcVis.put(rsrc, vis);
+ Path destPath = dirs.getLocalPathForWrite(
+ basedir.toString(), size, conf);
+ FSDownload fsd =
+ new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
+ destPath, rsrc, new Random(sharedSeed));
+ pending.put(rsrc, exec.submit(fsd));
+
+ try {
+ for (Map.Entry<LocalResource,Future<Path>> p : pending.entrySet()) {
+ p.getValue().get();
+ Assert.fail("We localized a file that is not public.");
+ }
+ } catch (ExecutionException e) {
+ Assert.assertTrue(e.getCause() instanceof IOException);
+ } finally {
+ exec.shutdown();
+ }
+ }
+
+ @Test
public void testDownload() throws IOException, URISyntaxException,
InterruptedException {
Configuration conf = new Configuration();
@@ -140,14 +188,9 @@ public class TestFSDownload {
int[] sizes = new int[10];
for (int i = 0; i < 10; ++i) {
sizes[i] = rand.nextInt(512) + 512;
- LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
- switch (i%3) {
- case 1:
- vis = LocalResourceVisibility.PRIVATE;
- break;
- case 2:
+ LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
+ if (i%2 == 1) {
vis = LocalResourceVisibility.APPLICATION;
- break;
}
Path p = new Path(basedir, "" + i);
LocalResource rsrc = createFile(files, p, sizes[i], rand, vis);
@@ -176,17 +219,8 @@ public class TestFSDownload {
System.out.println("File permission " + perm +
" for rsrc vis " + p.getKey().getVisibility().name());
assert(rsrcVis.containsKey(p.getKey()));
- switch (rsrcVis.get(p.getKey())) {
- case PUBLIC:
- Assert.assertTrue("Public file should be 555",
- perm.toShort() == FSDownload.PUBLIC_FILE_PERMS.toShort());
- break;
- case PRIVATE:
- case APPLICATION:
- Assert.assertTrue("Private file should be 500",
- perm.toShort() == FSDownload.PRIVATE_FILE_PERMS.toShort());
- break;
- }
+ Assert.assertTrue("Private file should be 500",
+ perm.toShort() == FSDownload.PRIVATE_FILE_PERMS.toShort());
}
} catch (ExecutionException e) {
throw new IOException("Failed exec", e);
@@ -250,14 +284,9 @@ public class TestFSDownload {
LocalDirAllocator dirs =
new LocalDirAllocator(TestFSDownload.class.getName());
for (int i = 0; i < 5; ++i) {
- LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
- switch (rand.nextInt()%3) {
- case 1:
- vis = LocalResourceVisibility.PRIVATE;
- break;
- case 2:
+ LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
+ if (i%2 == 1) {
vis = LocalResourceVisibility.APPLICATION;
- break;
}
Path p = new Path(basedir, "dir" + i + ".jar");