You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/04/21 20:06:41 UTC
svn commit: r1588947 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/common/FileUtils.java
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java
ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
Author: xuefu
Date: Mon Apr 21 18:06:41 2014
New Revision: 1588947
URL: http://svn.apache.org/r1588947
Log:
HIVE-6916: Export/import inherit permissions from parent directory (Szehon via Xuefu)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1588947&r1=1588946&r2=1588947&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Mon Apr 21 18:06:41 2014
@@ -31,6 +31,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
@@ -489,4 +491,32 @@ public final class FileUtils {
}
}
}
+
+ /**
+ * Copies files between filesystems.
+ */
+ public static boolean copy(FileSystem srcFS, Path src,
+ FileSystem dstFS, Path dst,
+ boolean deleteSource,
+ boolean overwrite,
+ HiveConf conf) throws IOException {
+ boolean copied = FileUtil.copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf);
+ boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
+ if (copied && inheritPerms) {
+ FileStatus destFileStatus = dstFS.getFileStatus(dst);
+ FsPermission perm = destFileStatus.getPermission();
+ String permString = Integer.toString(perm.toShort(), 8);
+ String group = destFileStatus.getGroup();
+ //use FsShell to change group and permissions recursively
+ try {
+ FsShell fshell = new FsShell();
+ fshell.setConf(conf);
+ fshell.run(new String[]{"-chgrp", "-R", group, dst.toString()});
+ fshell.run(new String[]{"-chmod", "-R", permString, dst.toString()});
+ } catch (Exception e) {
+ throw new IOException("Unable to set permissions of " + dst, e);
+ }
+ }
+ return copied;
+ }
}
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java?rev=1588947&r1=1588946&r2=1588947&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java Mon Apr 21 18:06:41 2014
@@ -223,6 +223,76 @@ public class TestFolderPermissions {
}
}
+ @Test
+ public void testEximPermissionInheritance() throws Exception {
+
+ //export the table to external file.
+ String myLocation = testDir + "/exim";
+ FileSystem fs = FileSystem.get(new URI(myLocation), conf);
+ fs.mkdirs(new Path(myLocation), FsPermission.createImmutable((short) 0777));
+
+ myLocation = myLocation + "/temp";
+
+ CommandProcessorResponse ret = driver.run("export table mysrc to '" + myLocation + "'");
+ Assert.assertEquals(0,ret.getResponseCode());
+
+ //check if exported data has inherited the permissions.
+ assertExistence(myLocation);
+ Assert.assertEquals(getPermissions(myLocation).toString(), "rwxrwxrwx");
+
+ assertExistence(myLocation + "/part1=1/part2=1");
+ Assert.assertEquals(getPermissions(myLocation + "/part1=1/part2=1").toString(), "rwxrwxrwx");
+ Assert.assertTrue(listChildrenPerms(myLocation + "/part1=1/part2=1").size() > 0);
+ for (FsPermission perm : listChildrenPerms(myLocation + "/part1=1/part2=1")) {
+ Assert.assertEquals("rwxrwxrwx", perm.toString());
+ }
+
+ assertExistence(myLocation + "/part1=2/part2=2");
+ Assert.assertEquals(getPermissions(myLocation + "/part1=2/part2=2").toString(), "rwxrwxrwx");
+ Assert.assertTrue(listChildrenPerms(myLocation + "/part1=2/part2=2").size() > 0);
+ for (FsPermission perm : listChildrenPerms(myLocation + "/part1=2/part2=2")) {
+ Assert.assertEquals("rwxrwxrwx", perm.toString());
+ }
+
+ //import the table back into another database
+ String testDb = "eximdb";
+ ret = driver.run("CREATE DATABASE " + testDb);
+ Assert.assertEquals(0,ret.getResponseCode());
+
+ //use another permission for this import location, to verify that it is really set.
+ assertExistence(testDir + "/" + testDb + ".db");
+ setPermissions(testDir + "/" + testDb + ".db", FsPermission.createImmutable((short) 0766));
+
+ ret = driver.run("USE " + testDb);
+ Assert.assertEquals(0,ret.getResponseCode());
+
+ ret = driver.run("import from '" + myLocation + "'");
+ Assert.assertEquals(0,ret.getResponseCode());
+
+ //check permissions of imported, from the exported table
+ assertExistence(testDir + "/" + testDb + ".db/mysrc");
+ Assert.assertEquals("rwxrw-rw-", getPermissions(testDir + "/" + testDb + ".db/mysrc").toString());
+
+ myLocation = testDir + "/" + testDb + ".db/mysrc";
+ assertExistence(myLocation);
+ Assert.assertEquals(getPermissions(myLocation).toString(), "rwxrw-rw-");
+
+ assertExistence(myLocation + "/part1=1/part2=1");
+ Assert.assertEquals(getPermissions(myLocation + "/part1=1/part2=1").toString(), "rwxrw-rw-");
+ Assert.assertTrue(listChildrenPerms(myLocation + "/part1=1/part2=1").size() > 0);
+ for (FsPermission perm : listChildrenPerms(myLocation + "/part1=1/part2=1")) {
+ Assert.assertEquals("rwxrw-rw-", perm.toString());
+ }
+
+ assertExistence(myLocation + "/part1=2/part2=2");
+ Assert.assertEquals(getPermissions(myLocation + "/part1=2/part2=2").toString(), "rwxrw-rw-");
+ Assert.assertTrue(listChildrenPerms(myLocation + "/part1=2/part2=2").size() > 0);
+ for (FsPermission perm : listChildrenPerms(myLocation + "/part1=2/part2=2")) {
+ Assert.assertEquals("rwxrw-rw-", perm.toString());
+ }
+ }
+
+
private void setPermissions(String locn, FsPermission permissions) throws Exception {
fs.setPermission(new Path(locn), permissions);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java?rev=1588947&r1=1588946&r2=1588947&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java Mon Apr 21 18:06:41 2014
@@ -24,8 +24,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer;
import org.apache.hadoop.hive.ql.plan.CopyWork;
@@ -69,7 +70,8 @@ public class CopyTask extends Task<CopyW
}
}
- if (!dstFs.mkdirs(toPath)) {
+ boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
+ if (!FileUtils.mkdir(dstFs, toPath, inheritPerms)) {
console.printError("Cannot make target directory: " + toPath.toString());
return 2;
}
@@ -77,8 +79,8 @@ public class CopyTask extends Task<CopyW
for (FileStatus oneSrc : srcs) {
console.printInfo("Copying file: " + oneSrc.getPath().toString());
LOG.debug("Copying file: " + oneSrc.getPath().toString());
- if (!FileUtil.copy(srcFs, oneSrc.getPath(), dstFs, toPath, false, // delete
- // source
+ if (!FileUtils.copy(srcFs, oneSrc.getPath(), dstFs, toPath,
+ false, // delete source
true, // overwrite destination
conf)) {
console.printError("Failed to copy: '" + oneSrc.getPath().toString()