You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/08/14 22:05:05 UTC
git commit: TEZ-1413. Fix build for
TestTezClientUtils.testLocalResourceVisibility (Prakash Ramachandran via
bikas)
Repository: tez
Updated Branches:
refs/heads/master 0af561228 -> 0557abf67
TEZ-1413. Fix build for TestTezClientUtils.testLocalResourceVisibility (Prakash Ramachandran via bikas)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0557abf6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0557abf6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0557abf6
Branch: refs/heads/master
Commit: 0557abf670e801958edb906b30046990ac27e20d
Parents: 0af5612
Author: Bikas Saha <bi...@apache.org>
Authored: Thu Aug 14 13:05:06 2014 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Thu Aug 14 13:05:06 2014 -0700
----------------------------------------------------------------------
.../apache/tez/client/TestTezClientUtils.java | 122 +++++++++++--------
.../apache/tez/common/TestTezCommonUtils.java | 11 +-
2 files changed, 76 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0557abf6/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index b346d16..c44db47 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -22,24 +22,26 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
-import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezUncheckedException;
import org.junit.Assert;
import org.junit.Assume;
-import org.junit.Ignore;
import org.junit.Test;
/**
@@ -222,71 +224,83 @@ public class TestTezClientUtils {
&& javaOpts.contains(TezConfiguration.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE));
}
- @Ignore @Test (timeout = 5000)
- public void testLocalResourceVisibility() throws Exception {
+ // To run this test case see TestTezCommonUtils::testLocalResourceVisibility
+ // We do not have much control over the directory structure, cannot mock as the functions are
+ // static and do not want to spin up a minitez cluster just for this.
+ public static void testLocalResourceVisibility(DistributedFileSystem remoteFs, Configuration conf)
+ throws Exception {
- Assume.assumeFalse(Shell.WINDOWS);
-
- File topLevelDir = null;
- File subDir = null;
- File publicFile = null;
- File privateFile = null;
- File subDirFile = null;
+ Path topLevelDir = null;
try {
- topLevelDir = createDirectoryOrFile(new File(TEST_ROOT_DIR), "testLRVisibility", true);
- subDir = createDirectoryOrFile(topLevelDir, "subDir", true);
- publicFile = createDirectoryOrFile(topLevelDir, "publicFile", false);
- privateFile = createDirectoryOrFile(topLevelDir, "privateFile", false);
- subDirFile = createDirectoryOrFile(subDir, "subDirFile", false);
+ FsPermission publicDirPerms = new FsPermission((short) 0755); // rwxr-xr-x
+ FsPermission privateDirPerms = new FsPermission((short) 0754); // rwxr-xr--
+ FsPermission publicFilePerms = new FsPermission((short) 0554); // r-xr-xr--
+ FsPermission privateFilePerms = new FsPermission((short) 0550); // r-xr-x---
- // no direct way to remove execute permission for others, set all to no execute and then
- // only the owner to execute. this does not work well on windows
- Assert.assertTrue(subDir.setExecutable(false, false));
- Assert.assertTrue(subDir.setExecutable(true));
+ String fsURI = remoteFs.getUri().toString();
- Assert.assertTrue(privateFile.setReadable(false, false));
- Assert.assertTrue(privateFile.setReadable(true));
+ topLevelDir = new Path(fsURI, "/testLRVisibility");
+ Assert.assertTrue(remoteFs.mkdirs(topLevelDir, publicDirPerms));
- String tezLibUris = topLevelDir.toURI().toString() + "," + subDir.toURI().toString();
+ Path publicSubDir = new Path(topLevelDir, "public_sub_dir");
+ Assert.assertTrue(remoteFs.mkdirs(publicSubDir, publicDirPerms));
- TezConfiguration conf = new TezConfiguration();
- conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris);
- Credentials credentials = new Credentials();
+ Path privateSubDir = new Path(topLevelDir, "private_sub_dir");
+ Assert.assertTrue(remoteFs.mkdirs(privateSubDir, privateDirPerms));
- Map<String, LocalResource> localResourceMap = TezClientUtils.setupTezJarsLocalResources(conf,
- credentials);
+ Path publicFile = new Path(publicSubDir, "public_file");
+ Assert.assertTrue(remoteFs.createNewFile(publicFile));
+ remoteFs.setPermission(publicFile, publicFilePerms);
- Assert.assertTrue(localResourceMap.get(publicFile.getName()).getVisibility() ==
- LocalResourceVisibility.PUBLIC);
+ Path privateFile = new Path(publicSubDir, "private_file");
+ Assert.assertTrue(remoteFs.createNewFile(privateFile));
+ remoteFs.setPermission(privateFile, privateFilePerms);
- // ancestor directories have execute perms but file does not have read perms for other users.
- Assert.assertTrue(localResourceMap.get(privateFile.getName()).getVisibility() ==
- LocalResourceVisibility.PRIVATE);
+ Path publicFileInPrivateSubdir = new Path(privateSubDir, "public_file_in_private_subdir");
+ Assert.assertTrue(remoteFs.createNewFile(publicFileInPrivateSubdir));
+ remoteFs.setPermission(publicFileInPrivateSubdir, publicFilePerms);
- // file has read perms but parent dir does not have execute perms for other users
- Assert.assertTrue(localResourceMap.get(subDirFile.getName()).getVisibility() ==
- LocalResourceVisibility.PRIVATE);
- } finally {
- for(File f : new File[] { subDirFile, privateFile, publicFile, subDir, topLevelDir }) {
- if (f != null) {
- f.delete();
- }
- }
- }
- }
+ TezConfiguration tezConf = new TezConfiguration(conf);
+ String tmpTezLibUris = String.format("%s,%s,%s,%s", topLevelDir, publicSubDir, privateSubDir,
+ conf.get(TezConfiguration.TEZ_LIB_URIS, ""));
+ tezConf.set(TezConfiguration.TEZ_LIB_URIS, tmpTezLibUris);
+
+ Map<String, LocalResource> lrMap =
+ TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials());
+
+ Assert.assertEquals(publicFile.getName(), LocalResourceVisibility.PUBLIC,
+ lrMap.get(publicFile.getName()).getVisibility());
- private File createDirectoryOrFile(File parent, String child, boolean isDir) throws IOException {
- File file = new File(parent, child);
- if (!file.exists()) {
- if (isDir) {
- Assert.assertTrue(String.format("creating dir %s", file.getPath()), file.mkdirs());
- file.setWritable(true);
- } else {
- Assert.assertTrue(String.format("creating file %s", file.getPath()), file.createNewFile());
+ Assert.assertEquals(privateFile.getName(), LocalResourceVisibility.PRIVATE,
+ lrMap.get(privateFile.getName()).getVisibility());
+
+ Assert.assertEquals(publicFileInPrivateSubdir.getName(), LocalResourceVisibility.PRIVATE,
+ lrMap.get(publicFileInPrivateSubdir.getName()).getVisibility());
+
+ // test tar.gz
+ tezConf = new TezConfiguration(conf);
+ Path tarFile = new Path(topLevelDir, "foo.tar.gz");
+ Assert.assertTrue(remoteFs.createNewFile(tarFile));
+
+ //public
+ remoteFs.setPermission(tarFile, publicFilePerms);
+ tezConf.set(TezConfiguration.TEZ_LIB_URIS, tarFile.toString());
+ lrMap = TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials());
+
+ Assert.assertEquals(LocalResourceVisibility.PUBLIC,
+ lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility());
+
+ //private
+ remoteFs.setPermission(tarFile, privateFilePerms);
+ lrMap = TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials());
+ Assert.assertEquals(LocalResourceVisibility.PRIVATE,
+ lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility());
+
+ } finally {
+ if (topLevelDir != null) {
+ remoteFs.delete(topLevelDir, true);
}
}
-
- return file;
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0557abf6/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
index f52f69b..79a35bb 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
@@ -27,10 +27,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.TestTezClientUtils;
import org.apache.tez.dag.api.TezConfiguration;
-import org.apache.tez.common.TezCommonUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -226,4 +224,11 @@ public class TestTezCommonUtils {
+ TezConfiguration.DAG_RECOVERY_SUMMARY_FILE_SUFFIX;
Assert.assertEquals(summaryRecoveryPathj.toString(), expectedDir);
}
+
+ // This test is running here to leverage existing mini cluster
+ @Test
+ public void testLocalResourceVisibility() throws Exception {
+ TestTezClientUtils.testLocalResourceVisibility(dfsCluster.getFileSystem(), conf);
+ }
+
}