You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2016/03/02 02:53:02 UTC
incubator-sentry git commit: SENTRY-1087:Capture URI when using Hive
Serdes (Hao Hao, Reviewed by: Sravya Tirukkovalur and Lenni Kuff)
Repository: incubator-sentry
Updated Branches:
refs/heads/upstream-master [created] 34b5afc36
SENTRY-1087:Capture URI when using Hive Serdes (Hao Hao, Reviewed by: Sravya Tirukkovalur and Lenni Kuff)
Change-Id: I06d19ffc8e5dfc8fd16c6c5a79ea40270580fb72
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/34b5afc3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/34b5afc3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/34b5afc3
Branch: refs/heads/upstream-master
Commit: 34b5afc36feaaea19ea129e985c0cad95629c4e5
Parents: f727a0c
Author: hahao <ha...@cloudera.com>
Authored: Tue Mar 1 17:18:29 2016 -0800
Committer: hahao <ha...@cloudera.com>
Committed: Tue Mar 1 17:51:39 2016 -0800
----------------------------------------------------------------------
.../binding/hive/HiveAuthzBindingHook.java | 92 ++++++++++++++-
.../hive/authz/HiveAuthzPrivilegesMap.java | 3 +-
.../sentry/binding/hive/conf/HiveAuthzConf.java | 8 ++
.../e2e/hive/TestCustomSerdePrivileges.java | 115 +++++++++++++++++++
.../e2e/hive/TestPrivilegesAtFunctionScope.java | 34 ++++++
5 files changed, 250 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
index 08c0e98..dd33d2d 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
+import java.util.Arrays;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hive.common.JavaUtils;
@@ -44,6 +45,7 @@ import org.apache.hadoop.hive.ql.hooks.Entity.Type;
import org.apache.hadoop.hive.ql.hooks.Hook;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.AuthorizationException;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.AbstractSemanticAnalyzerHook;
@@ -87,9 +89,12 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook {
private Database currDB = Database.ALL;
private Table currTab;
private AccessURI udfURI;
+ private AccessURI serdeURI;
private AccessURI partitionURI;
private Table currOutTab = null;
private Database currOutDB = null;
+ private final List<String> serdeWhiteList;
+ private boolean serdeURIPrivilegesEnabled;
// True if this is a basic DESCRIBE <table> operation. False for other DESCRIBE variants
// like DESCRIBE [FORMATTED|EXTENDED]. Required because Hive treats these stmts as the same
@@ -113,6 +118,12 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook {
authzConf = loadAuthzConf(hiveConf);
hiveAuthzBinding = new HiveAuthzBinding(hiveConf, authzConf);
+ String serdeWhiteLists = authzConf.get(HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST,
+ HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST_DEFAULT);
+ serdeWhiteList = Arrays.asList(serdeWhiteLists.split(","));
+ serdeURIPrivilegesEnabled = authzConf.getBoolean(HiveAuthzConf.HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED,
+ HiveAuthzConf.HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT);
+
FunctionRegistry.setupPermissionsForBuiltinUDFs("", HiveAuthzConf.HIVE_UDF_BLACK_LIST);
}
@@ -164,6 +175,16 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook {
currDB = new Database(BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(0).getText()));
break;
case HiveParser.TOK_CREATETABLE:
+
+ for (Node childNode : ast.getChildren()) {
+ ASTNode childASTNode = (ASTNode) childNode;
+ if ("TOK_TABLESERIALIZER".equals(childASTNode.getText())) {
+ ASTNode serdeNode = (ASTNode)childASTNode.getChild(0);
+ String serdeClassName = BaseSemanticAnalyzer.unescapeSQLString(serdeNode.getChild(0).getText());
+ setSerdeURI(serdeClassName);
+ }
+ }
+
case HiveParser.TOK_CREATEVIEW:
/*
* Compiler doesn't create read/write entities for create table.
@@ -283,7 +304,18 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook {
currOutDB = extractDatabase((ASTNode) ast.getChild(0));
currOutTab = extractTable((ASTNode) ast.getChild(0).getChild(0).getChild(0));
break;
- default:
+ case HiveParser.TOK_ALTERTABLE:
+
+ for (Node childNode : ast.getChildren()) {
+ ASTNode childASTNode = (ASTNode) childNode;
+ if ("TOK_ALTERTABLE_SERIALIZER".equals(childASTNode.getText())) {
+ ASTNode serdeNode = (ASTNode)childASTNode.getChild(0);
+ String serdeClassName = BaseSemanticAnalyzer.unescapeSQLString(serdeNode.getText());
+ setSerdeURI(serdeClassName);
+ }
+ }
+
+ default:
currDB = getCanonicalDb();
break;
}
@@ -497,6 +529,13 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook {
outputHierarchy.add(dbHierarchy);
getInputHierarchyFromInputs(inputHierarchy, inputs);
+
+ if (serdeURI != null) {
+ List<DBModelAuthorizable> serdeUriHierarchy = new ArrayList<DBModelAuthorizable>();
+ serdeUriHierarchy.add(hiveAuthzBinding.getAuthServer());
+ serdeUriHierarchy.add(serdeURI);
+ outputHierarchy.add(serdeUriHierarchy);
+ }
break;
case TABLE:
// workaround for add partitions
@@ -535,6 +574,14 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook {
externalAuthorizableHierarchy.add(currOutTab);
outputHierarchy.add(externalAuthorizableHierarchy);
}
+
+ if (serdeURI != null) {
+ List<DBModelAuthorizable> serdeUriHierarchy = new ArrayList<DBModelAuthorizable>();
+ serdeUriHierarchy.add(hiveAuthzBinding.getAuthServer());
+ serdeUriHierarchy.add(serdeURI);
+ outputHierarchy.add(serdeUriHierarchy);
+ }
+
break;
case FUNCTION:
/* The 'FUNCTION' privilege scope currently used for
@@ -956,4 +1003,47 @@ public class HiveAuthzBindingHook extends AbstractSemanticAnalyzerHook {
throw new SemanticException(e);
}
}
+
+ private static boolean hasPrefixMatch(List<String> prefixList, final String str) {
+ for (String prefix : prefixList) {
+ if (str.startsWith(prefix)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Set the Serde URI privileges. If the URI privileges are not set, which serdeURI will be null,
+ * the URI authorization checks will be skipped.
+ */
+ private void setSerdeURI(String serdeClassName) throws SemanticException {
+ if (!serdeURIPrivilegesEnabled) {
+ return;
+ }
+
+ // WhiteList Serde Jar can be used by any users. WhiteList checking is
+ // done by comparing the Java package name. The assumption is cluster
+ // admin will ensure there is no Java namespace collision.
+ // e.g org.apache.hadoop.hive.serde2 is used by hive and cluster admin should
+ // ensure no custom Serde class is introduced under the same namespace.
+ if (!hasPrefixMatch(serdeWhiteList, serdeClassName)) {
+ try {
+ CodeSource serdeSrc = Class.forName(serdeClassName, true, Utilities.getSessionSpecifiedClassLoader()).getProtectionDomain().getCodeSource();
+ if (serdeSrc == null) {
+ throw new SemanticException("Could not resolve the jar for Serde class " + serdeClassName);
+ }
+
+ String serdeJar = serdeSrc.getLocation().getPath();
+ if (serdeJar == null || serdeJar.isEmpty()) {
+ throw new SemanticException("Could not find the jar for Serde class " + serdeClassName + "to validate privileges");
+ }
+
+ serdeURI = parseURI(serdeSrc.getLocation().toString(), true);
+ } catch (ClassNotFoundException e) {
+ throw new SemanticException("Error retrieving Serde class:" + e.getMessage(), e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java
index 0c3bee3..8e70492 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/authz/HiveAuthzPrivilegesMap.java
@@ -39,6 +39,7 @@ public class HiveAuthzPrivilegesMap {
HiveAuthzPrivileges tableCreatePrivilege = new HiveAuthzPrivileges.AuthzPrivilegeBuilder().
addOutputObjectPriviledge(AuthorizableType.Db, EnumSet.of(DBModelAction.CREATE)).
addInputObjectPriviledge(AuthorizableType.URI, EnumSet.of(DBModelAction.ALL)).//TODO: make it optional
+ addOutputObjectPriviledge(AuthorizableType.URI, EnumSet.of(DBModelAction.ALL)).
setOperationScope(HiveOperationScope.DATABASE).
setOperationType(HiveOperationType.DDL).
build();
@@ -225,7 +226,6 @@ public class HiveAuthzPrivilegesMap {
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERPARTITION_PROTECTMODE, alterTablePrivilege);
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERPARTITION_SERDEPROPERTIES, alterTablePrivilege);
- hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_SERIALIZER, alterTablePrivilege);
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_MERGEFILES, alterTablePrivilege);
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_SKEWED, alterTablePrivilege);
@@ -238,6 +238,7 @@ public class HiveAuthzPrivilegesMap {
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_ADDPARTS, addPartitionPrivilege);
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_RENAME, alterTableRenamePrivilege);
+ hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_SERIALIZER, alterTableAndUriPrivilege);
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTABLE_LOCATION, alterTableAndUriPrivilege);
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERPARTITION_LOCATION, alterTableAndUriPrivilege);
hiveAuthzStmtPrivMap.put(HiveOperation.ALTERTBLPART_SKEWED_LOCATION, alterTableAndUriPrivilege);//TODO: Needs test case
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
index 6b79dda..1093a09 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
@@ -51,6 +51,13 @@ public class HiveAuthzConf extends Configuration {
public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT =
"set,reset,reload";
+ public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist";
+ public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2";
+
+ // Disable the serde Uri privileges by default for backward compatibilities.
+ public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges";
+ public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false;
+
public static final String HIVE_UDF_WHITE_LIST =
"concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," +
"ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," +
@@ -76,6 +83,7 @@ public class HiveAuthzConf extends Configuration {
"noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath";
public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method";
+
/**
* Config setting definitions
*/
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCustomSerdePrivileges.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCustomSerdePrivileges.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCustomSerdePrivileges.java
new file mode 100644
index 0000000..6dfdb3c
--- /dev/null
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestCustomSerdePrivileges.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.tests.e2e.hive;
+
+import com.google.common.collect.Maps;
+import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
+import org.apache.sentry.provider.file.PolicyFile;
+import org.junit.*;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.security.CodeSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Map;
+
+public class TestCustomSerdePrivileges extends AbstractTestWithHiveServer {
+ private static Context context;
+ private static Map<String, String> properties;
+ private PolicyFile policyFile;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ properties = Maps.newHashMap();
+
+ // Start the Hive Server without buildin Serde, such as
+ // "org.apache.hadoop.hive.serde2.OpenCSVSerde". Instead,
+ // used a bogus class name for testing.
+ properties.put(HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST, "org.example.com");
+ properties.put(HiveAuthzConf.HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED, "true");
+ context = createContext(properties);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if(context != null) {
+ context.close();
+ }
+ }
+
+ @Before
+ public void setupPolicyFile() throws Exception {
+ policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP);
+ }
+
+ /**
+ * User with db level access and Uri privileges on the Serde Jar should be able
+ * to create tables with Serde.
+ * User with db level access but without Uri privileges on the Serde Jar will fail
+ * on creating tables with Serde.
+ */
+ @Test
+ public void testSerdePrivilegesWithoutBuildinJar() throws Exception {
+ String db = "db1";
+ String tableName1 = "tab1";
+
+ String serdeClassName = "org.apache.hadoop.hive.serde2.OpenCSVSerde";
+ CodeSource serdeSrc = Class.forName(serdeClassName).getProtectionDomain().getCodeSource();
+ String serdeLocation = serdeSrc.getLocation().getPath();
+
+ policyFile
+ .addRolesToGroup(USERGROUP1, "db1_all")
+ .addRolesToGroup(USERGROUP2, "db1_all", "SERDE_JAR")
+ .addPermissionsToRole("db1_all", "server=server1->db=" + db)
+ .addPermissionsToRole("db1_tab1", "server=server1->db=" + db + "->table=" + tableName1)
+ .addPermissionsToRole("SERDE_JAR", "server=server1->uri=file://" + serdeLocation)
+ .setUserGroupMapping(StaticUserGroup.getStaticMapping());
+ policyFile.write(context.getPolicyFile());
+
+ Connection connection = context.createConnection(ADMIN1);
+ Statement statement = context.createStatement(connection);
+ statement.execute("DROP DATABASE IF EXISTS " + db + " CASCADE");
+ statement.execute("CREATE DATABASE " + db);
+ context.close();
+
+ // User1 does not have the URI privileges to use the Serde Jar.
+ // The table creation will fail.
+ connection = context.createConnection(USER1_1);
+ statement = context.createStatement(connection);
+ statement.execute("USE " + db);
+ try {
+ statement.execute("create table " + db + "." + tableName1 + " (a string, b string) " +
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' " + " STORED AS TEXTFILE");
+ Assert.fail("Expect create table with Serde to fail");
+ } catch (SQLException e) {
+ context.verifyAuthzException(e);
+ }
+ context.close();
+
+ // User2 has the URI privileges to use the Serde Jar.
+ // The table creation will succeed.
+ connection = context.createConnection(USER2_1);
+ statement = context.createStatement(connection);
+ statement.execute("USE " + db);
+ statement.execute("create table " + db + "." + tableName1 + " (a string, b string) ROW FORMAT" +
+ " SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' " + " STORED AS TEXTFILE");
+ context.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/34b5afc3/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
index cfaf7c3..bb8d61d 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
@@ -225,4 +225,38 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu
statement.close();
connection.close();
}
+
+ /**
+ * User with db level access should be able to create/alter tables with buildin Serde.
+ */
+ @Test
+ public void testSerdePrivileges() throws Exception {
+ String tableName1 = "tab1";
+ String tableName2 = "tab2";
+
+ Connection connection = context.createConnection(ADMIN1);
+ Statement statement = context.createStatement(connection);
+ statement.execute("DROP DATABASE IF EXISTS " + DB1 + " CASCADE");
+ statement.execute("CREATE DATABASE " + DB1);
+
+ context.close();
+
+ policyFile
+ .addRolesToGroup(USERGROUP1, "db1_all")
+ .addPermissionsToRole("db1_all", "server=server1->db=" + DB1);
+ writePolicyFile(policyFile);
+
+ connection = context.createConnection(USER1_1);
+ statement = context.createStatement(connection);
+ statement.execute("USE " + DB1);
+ statement.execute("create table " + DB1 + "." + tableName1
+ + " (a string, b string) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' "
+ + " STORED AS TEXTFILE");
+
+ statement.execute("create table " + DB1 + "." + tableName2 + " (a string, b string)");
+ statement.execute("alter table " + DB1 + "." + tableName2
+ + " SET SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'");
+
+ context.close();
+ }
}