You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by sd...@apache.org on 2015/08/14 09:29:02 UTC
[28/50] [abbrv] incubator-sentry git commit: SENTRY-755: HDFS access
of data files should be disabled for user with privileges only on some
columns (Sravya Tirukkovalur, Reviewed by: Lenni Kuff)
SENTRY-755: HDFS access of data files should be disabled for user with privileges only on some columns (Sravya Tirukkovalur, Reviewed by: Lenni Kuff)
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/a5b37c7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/a5b37c7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/a5b37c7e
Branch: refs/heads/hive_plugin_v2
Commit: a5b37c7e122d0126a4d2a4f57ecf0359feadf0d5
Parents: 100e239
Author: Sravya Tirukkovalur <sr...@clouera.com>
Authored: Fri Jul 24 13:35:23 2015 -0700
Committer: Sravya Tirukkovalur <sr...@clouera.com>
Committed: Fri Jul 24 15:44:53 2015 -0700
----------------------------------------------------------------------
.../hdfs/SentryAuthorizationProvider.java | 10 +-
.../org/apache/sentry/hdfs/SentryPlugin.java | 8 +-
.../SentryPolicyServiceClientDefaultImpl.java | 8 +-
.../tests/e2e/hdfs/TestHDFSIntegration.java | 138 ++++++++++++++++++-
4 files changed, 153 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/a5b37c7e/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java
index f3d8aac..d167183 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryAuthorizationProvider.java
@@ -300,7 +300,15 @@ public class SentryAuthorizationProvider
builder.setName(null);
return list;
}
-
+ /*
+ Returns hadoop acls if
+ - Not managed
+ - Not stale and not an auth obj
+ Returns hive:hive
+ - If stale
+ Returns sentry acls
+ - Otherwise, if not stale and auth obj
+ */
@Override
public AclFeature getAclFeature(INodeAuthorizationInfo node, int snapshotId) {
AclFeature f = null;
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/a5b37c7e/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
----------------------------------------------------------------------
diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
index 221c397..7587a1d 100644
--- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
+++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java
@@ -165,7 +165,9 @@ public class SentryPlugin implements SentryPolicyStorePlugin {
if (request.isSetPrivileges()) {
String roleName = request.getRoleName();
for (TSentryPrivilege privilege : request.getPrivileges()) {
- onAlterSentryRoleGrantPrivilegeCore(roleName, privilege);
+ if(!("COLUMN".equalsIgnoreCase(privilege.getPrivilegeScope()))) {
+ onAlterSentryRoleGrantPrivilegeCore(roleName, privilege);
+ }
}
}
}
@@ -202,7 +204,9 @@ public class SentryPlugin implements SentryPolicyStorePlugin {
if (request.isSetPrivileges()) {
String roleName = request.getRoleName();
for (TSentryPrivilege privilege : request.getPrivileges()) {
- onAlterSentryRoleRevokePrivilegeCore(roleName, privilege);
+ if(!("COLUMN".equalsIgnoreCase(privilege.getPrivilegeScope()))) {
+ onAlterSentryRoleRevokePrivilegeCore(roleName, privilege);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/a5b37c7e/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
index c3c1907..533a28c 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
@@ -529,7 +529,7 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService
ImmutableList.Builder<String> listBuilder = ImmutableList.builder();
listBuilder.add(columnName);
revokePrivilege(requestorUserName, roleName,
- PrivilegeScope.TABLE, server, null,
+ PrivilegeScope.COLUMN, server, null,
db, table, listBuilder.build(), action);
}
@@ -539,7 +539,7 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService
ImmutableList.Builder<String> listBuilder = ImmutableList.builder();
listBuilder.add(columnName);
revokePrivilege(requestorUserName, roleName,
- PrivilegeScope.TABLE, server, null,
+ PrivilegeScope.COLUMN, server, null,
db, table, listBuilder.build(), action, grantOption);
}
@@ -547,7 +547,7 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService
String server, String db, String table, List<String> columns, String action)
throws SentryUserException {
revokePrivilege(requestorUserName, roleName,
- PrivilegeScope.TABLE, server, null,
+ PrivilegeScope.COLUMN, server, null,
db, table, columns, action);
}
@@ -555,7 +555,7 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService
String server, String db, String table, List<String> columns, String action, Boolean grantOption)
throws SentryUserException {
revokePrivilege(requestorUserName, roleName,
- PrivilegeScope.TABLE, server, null,
+ PrivilegeScope.COLUMN, server, null,
db, table, columns, action, grantOption);
}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/a5b37c7e/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
index 35a9213..786150b 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hdfs/TestHDFSIntegration.java
@@ -105,6 +105,7 @@ public class TestHDFSIntegration {
private static final Logger LOGGER = LoggerFactory
.getLogger(TestHDFSIntegration.class);
+
public static class WordCountMapper extends MapReduceBase implements
Mapper<LongWritable, Text, String, Long> {
@@ -149,6 +150,8 @@ public class TestHDFSIntegration {
protected static SentrySrv sentryServer;
protected static boolean testSentryHA = false;
private static final long STALE_THRESHOLD = 5000;
+ private static final long CACHE_REFRESH = 100; //Default is 500, but we want it to be low
+ // in our tests so that changes reflect soon
private static String fsURI;
private static int hmsPort;
@@ -273,9 +276,9 @@ public class TestHDFSIntegration {
out.close();
Reflection.staticField("hiveSiteURL")
- .ofType(URL.class)
- .in(HiveConf.class)
- .set(hiveSite.toURI().toURL());
+ .ofType(URL.class)
+ .in(HiveConf.class)
+ .set(hiveSite.toURI().toURL());
metastore = new InternalMetastoreServer(hiveConf);
new Thread() {
@@ -361,6 +364,8 @@ public class TestHDFSIntegration {
conf.set("sentry.authorization-provider.hdfs-path-prefixes", "/user/hive/warehouse,/tmp/external");
conf.set("sentry.authorization-provider.cache-refresh-retry-wait.ms", "5000");
+ conf.set("sentry.authorization-provider.cache-refresh-interval.ms", String.valueOf(CACHE_REFRESH));
+
conf.set("sentry.authorization-provider.cache-stale-threshold.ms", String.valueOf(STALE_THRESHOLD));
conf.set("sentry.hdfs.service.security.mode", "none");
@@ -486,7 +491,7 @@ public class TestHDFSIntegration {
conn = hiveServer2.createConnection("hive", "hive");
stmt = conn.createStatement();
for( String role:roles) {
- stmt.execute("drop role " + role);
+ stmt.execute("drop role " + role);
}
stmt.close();
conn.close();
@@ -911,6 +916,114 @@ public class TestHDFSIntegration {
}
+ @Test
+ public void testColumnPrivileges() throws Throwable {
+ String dbName = "db2";
+
+ tmpHDFSDir = new Path("/tmp/external");
+ dbNames = new String[]{dbName};
+ roles = new String[]{"admin_role", "tab_role", "db_role", "col_role"};
+ admin = StaticUserGroup.ADMIN1;
+
+ Connection conn;
+ Statement stmt;
+
+ conn = hiveServer2.createConnection("hive", "hive");
+ stmt = conn.createStatement();
+ stmt.execute("create role admin_role");
+ stmt.execute("grant all on server server1 to role admin_role with grant option");
+ stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);
+
+ conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
+ stmt = conn.createStatement();
+ stmt.execute("create database " + dbName);
+ stmt.execute("use "+ dbName);
+ stmt.execute("create table p1 (s string) partitioned by (month int, day int)");
+ stmt.execute("alter table p1 add partition (month=1, day=1)");
+ stmt.execute("alter table p1 add partition (month=1, day=2)");
+ stmt.execute("alter table p1 add partition (month=2, day=1)");
+ stmt.execute("alter table p1 add partition (month=2, day=2)");
+ loadData(stmt);
+
+ stmt.execute("create role db_role");
+ stmt.execute("grant select on database " + dbName + " to role db_role");
+ stmt.execute("create role tab_role");
+ stmt.execute("grant select on p1 to role tab_role");
+ stmt.execute("create role col_role");
+ stmt.execute("grant select(s) on p1 to role col_role");
+
+ stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP1);
+
+ stmt.execute("grant role tab_role to group "+ StaticUserGroup.USERGROUP2);
+ stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP2);
+
+ stmt.execute("grant role db_role to group "+ StaticUserGroup.USERGROUP3);
+ stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP3);
+
+ stmt.execute("grant role col_role to group " + StaticUserGroup.ADMINGROUP);
+
+ Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
+
+ //User with just column level privileges cannot read HDFS
+ verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.USERGROUP1, false);
+
+ //User with permissions on table and column can read HDFS file
+ verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true);
+
+ //User with permissions on db and column can read HDFS file
+ verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP3, true);
+
+ //User with permissions on server and column cannot read HDFS file
+ //TODO:SENTRY-751
+ verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.ADMINGROUP, false);
+
+ stmt.close();
+ conn.close();
+
+ }
+
+ /*
+ TODO:SENTRY-819
+ */
+ @Test
+ public void testAllColumn() throws Throwable {
+ String dbName = "db2";
+
+ tmpHDFSDir = new Path("/tmp/external");
+ dbNames = new String[]{dbName};
+ roles = new String[]{"admin_role", "col_role"};
+ admin = StaticUserGroup.ADMIN1;
+
+ Connection conn;
+ Statement stmt;
+
+ conn = hiveServer2.createConnection("hive", "hive");
+ stmt = conn.createStatement();
+ stmt.execute("create role admin_role");
+ stmt.execute("grant all on server server1 to role admin_role with grant option");
+ stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);
+
+ conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
+ stmt = conn.createStatement();
+ stmt.execute("create database " + dbName);
+ stmt.execute("use "+ dbName);
+ stmt.execute("create table p1 (c1 string, c2 string) partitioned by (month int, day int)");
+ stmt.execute("alter table p1 add partition (month=1, day=1)");
+ loadDataTwoCols(stmt);
+
+ stmt.execute("create role col_role");
+ stmt.execute("grant select(c1,c2) on p1 to role col_role");
+ stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP1);
+ Thread.sleep(100);
+
+ //User with privileges on all columns of the data cannot still read the HDFS files
+ verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.USERGROUP1, false);
+
+ stmt.close();
+ conn.close();
+
+ }
+
private void verifyQuery(Statement stmt, String table, int n) throws Throwable {
verifyQuery(stmt, table, n, NUM_RETRIES);
}
@@ -956,6 +1069,23 @@ public class TestHDFSIntegration {
rs.close();
}
+ private void loadDataTwoCols(Statement stmt) throws IOException, SQLException {
+ FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt"));
+ f1.writeChars("m1d1_t1, m1d1_t2\n");
+ f1.writeChars("m1d1_t2, m1d1_t2\n");
+ f1.writeChars("m1d1_t3, m1d1_t2\n");
+ f1.flush();
+ f1.close();
+ stmt.execute("load data inpath \'/tmp/f2.txt\' overwrite into table p1 partition (month=1, day=1)");
+ ResultSet rs = stmt.executeQuery("select * from p1");
+ List<String> vals = new ArrayList<String>();
+ while (rs.next()) {
+ vals.add(rs.getString(1));
+ }
+ Assert.assertEquals(3, vals.size());
+ rs.close();
+ }
+
private void writeToPath(String path, int numRows, String user, String group) throws IOException {
Path p = new Path(path);
miniDFS.getFileSystem().mkdirs(p);