You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by ha...@apache.org on 2017/01/12 19:06:52 UTC
sentry git commit: SENTRY-1577: Support "create function using jar"
for hive when Sentry is enabled (Aihua Xu, Reviewed by: Hao Hao,
Alexander Kolbasov)
Repository: sentry
Updated Branches:
refs/heads/master aba45daf7 -> 58d76b93a
SENTRY-1577: Support "create function using jar" for hive when Sentry is enabled (Aihua Xu, Reviewed by: Hao Hao, Alexander Kolbasov)
Change-Id: I1862b5b6ac919d2d3ccf8179b7710a8df24a09e7
Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/58d76b93
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/58d76b93
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/58d76b93
Branch: refs/heads/master
Commit: 58d76b93a0765f906ac4246d0a252baa8c807bae
Parents: aba45da
Author: hahao <ha...@cloudera.com>
Authored: Wed Jan 11 14:25:37 2017 -0800
Committer: hahao <ha...@cloudera.com>
Committed: Thu Jan 12 11:06:35 2017 -0800
----------------------------------------------------------------------
.../binding/hive/HiveAuthzBindingHookBase.java | 38 ++++++++++--
.../hive/SentryOnFailureHookContext.java | 5 +-
.../hive/SentryOnFailureHookContextImpl.java | 13 ++--
.../binding/hive/v2/HiveAuthzBindingHookV2.java | 14 ++++-
.../binding/hive/HiveAuthzBindingHook.java | 14 ++++-
sentry-tests/data/xudf.jar | Bin 0 -> 2430 bytes
.../e2e/hive/TestPrivilegesAtFunctionScope.java | 60 +++++++++++++++++--
7 files changed, 124 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
index d706b83..c4daea1 100644
--- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
+++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHookBase.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.sentry.binding.hive.authz.HiveAuthzBinding;
import org.apache.sentry.binding.hive.authz.HiveAuthzPrivileges;
@@ -76,6 +77,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerHook {
@@ -85,7 +87,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
protected final HiveAuthzConf authzConf;
protected Database currDB = Database.ALL;
protected Table currTab;
- protected AccessURI udfURI;
+ protected List<AccessURI> udfURIs;
protected AccessURI serdeURI;
protected AccessURI partitionURI;
protected Table currOutTab = null;
@@ -120,6 +122,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
throw new IllegalStateException("Session HiveConf is null");
}
authzConf = loadAuthzConf(hiveConf);
+ udfURIs = Lists.newArrayList();
hiveAuthzBinding = new HiveAuthzBinding(hiveConf, authzConf);
String serdeWhiteLists =
authzConf.get(HiveAuthzConf.HIVE_SENTRY_SERDE_WHITELIST,
@@ -174,7 +177,7 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
HiveOperation hiveOp, AuthorizationException e) {
SentryOnFailureHookContext hookCtx = new SentryOnFailureHookContextImpl(
context.getCommand(), context.getInputs(), context.getOutputs(),
- hiveOp, currDB, currTab, udfURI, null, context.getUserName(),
+ hiveOp, currDB, currTab, udfURIs, null, context.getUserName(),
context.getIpAddress(), e, context.getConf());
String csHooks = authzConf.get(
HiveAuthzConf.AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), "").trim();
@@ -188,6 +191,33 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
}
}
+ /**
+ * The command 'create function ... using jar <jar resources>' can create a function
+ * with the supplied jar resources in the command, which is translated into ASTNode being
+ * [functionName functionClass resourceList] and resourceList being [resourceType resourcePath].
+ * This function collects all the jar paths for the supplied jar resources.
+ *
+ * @param ast the AST node for the command
+ * @return the jar path list if any or an empty list
+ */
+ protected List<String> getFunctionJars(ASTNode ast) {
+ ASTNode resourcesNode = (ASTNode) ast.getFirstChildWithType(HiveParser.TOK_RESOURCE_LIST);
+
+ List<String> resources = new ArrayList<String>();
+ if (resourcesNode != null) {
+ for (int idx = 0; idx < resourcesNode.getChildCount(); ++idx) {
+ ASTNode resNode = (ASTNode) resourcesNode.getChild(idx);
+ ASTNode resTypeNode = (ASTNode) resNode.getChild(0);
+ ASTNode resUriNode = (ASTNode) resNode.getChild(1);
+ if (resTypeNode.getType() == HiveParser.TOK_JAR) {
+ resources.add(PlanUtils.stripQuotes(resUriNode.getText()));
+ }
+ }
+ }
+
+ return resources;
+ }
+
@VisibleForTesting
protected static AccessURI extractPartition(ASTNode ast) throws SemanticException {
for (int i = 0; i < ast.getChildCount(); i++) {
@@ -386,10 +416,10 @@ public abstract class HiveAuthzBindingHookBase extends AbstractSemanticAnalyzerH
* - CREATE TEMP FUNCTION
* - DROP TEMP FUNCTION.
*/
- if (udfURI != null) {
+ if (!udfURIs.isEmpty()) {
List<DBModelAuthorizable> udfUriHierarchy = new ArrayList<DBModelAuthorizable>();
udfUriHierarchy.add(hiveAuthzBinding.getAuthServer());
- udfUriHierarchy.add(udfURI);
+ udfUriHierarchy.addAll(udfURIs);
inputHierarchy.add(udfUriHierarchy);
for (WriteEntity writeEntity : outputs) {
List<DBModelAuthorizable> entityHierarchy = new ArrayList<DBModelAuthorizable>();
http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
index c101a4f..ac14fb1 100644
--- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
+++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContext.java
@@ -18,6 +18,7 @@
package org.apache.sentry.binding.hive;
+import java.util.List;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -76,9 +77,9 @@ public interface SentryOnFailureHookContext {
Table getTable();
/**
- * @return the udf URI
+ * @return the udf URIs
*/
- AccessURI getUdfURI();
+ List<AccessURI> getUdfURIs();
/**
* @return the partition URI
http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
index f97d7f3..15bfbea 100644
--- a/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
+++ b/sentry-binding/sentry-binding-hive-common/src/main/java/org/apache/sentry/binding/hive/SentryOnFailureHookContextImpl.java
@@ -18,6 +18,7 @@
package org.apache.sentry.binding.hive;
+import java.util.List;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -39,14 +40,14 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex
private final String ipAddress;
private final Database database;
private final Table table;
- private final AccessURI udfURI;
+ private final List<AccessURI> udfURIs;
private final AccessURI partitionURI;
private final AuthorizationException authException;
private final Configuration conf;
public SentryOnFailureHookContextImpl(String command,
Set<ReadEntity> inputs, Set<WriteEntity> outputs, HiveOperation hiveOp,
- Database db, Table tab, AccessURI udfURI, AccessURI partitionURI,
+ Database db, Table tab, List<AccessURI> udfURIs, AccessURI partitionURI,
String userName, String ipAddress, AuthorizationException e,
Configuration conf) {
this.command = command;
@@ -57,7 +58,7 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex
this.ipAddress = ipAddress;
this.database = db;
this.table = tab;
- this.udfURI = udfURI;
+ this.udfURIs = udfURIs;
this.partitionURI = partitionURI;
this.authException = e;
this.conf = conf;
@@ -104,8 +105,8 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex
}
@Override
- public AccessURI getUdfURI() {
- return udfURI;
+ public List<AccessURI> getUdfURIs() {
+ return udfURIs;
}
@Override
@@ -122,4 +123,4 @@ public class SentryOnFailureHookContextImpl implements SentryOnFailureHookContex
public Configuration getConf() {
return conf;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java b/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
index 91a2507..018ebf3 100644
--- a/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
+++ b/sentry-binding/sentry-binding-hive-v2/src/main/java/org/apache/sentry/binding/hive/v2/HiveAuthzBindingHookV2.java
@@ -69,10 +69,20 @@ public class HiveAuthzBindingHookV2 extends HiveAuthzBindingHookBase {
throw new SemanticException("Could not find the jar for UDF class " + udfClassName +
"to validate privileges");
}
- udfURI = parseURI(udfSrc.getLocation().toString(), true);
+ udfURIs.add(parseURI(udfSrc.getLocation().toString(), true));
} catch (ClassNotFoundException e) {
- throw new SemanticException("Error retrieving udf class", e);
+ List<String> functionJars = getFunctionJars(ast);
+ if (functionJars.isEmpty()) {
+ throw new SemanticException("Error retrieving udf class", e);
+ } else {
+ // Add the jars from the command "Create function using jar" to the access list
+ // Defer to hive to check if the class is in the jars
+ for(String jar : functionJars) {
+ udfURIs.add(parseURI(jar, false));
+ }
+ }
}
+
// create/drop function is allowed with any database
currDB = Database.ALL;
break;
http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
index 98c7eb0..f41960f 100644
--- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
+++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java
@@ -208,10 +208,20 @@ public class HiveAuthzBindingHook extends HiveAuthzBindingHookBase {
throw new SemanticException("Could not find the jar for UDF class " + udfClassName +
"to validate privileges");
}
- udfURI = parseURI(udfSrc.getLocation().toString(), true);
+ udfURIs.add(parseURI(udfSrc.getLocation().toString(), true));
} catch (ClassNotFoundException e) {
- throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e);
+ List<String> functionJars = getFunctionJars(ast);
+ if (functionJars.isEmpty()) {
+ throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e);
+ } else {
+ // Add the jars from the command "Create function using jar" to the access list
+ // Defer to hive to check if the class is in the jars
+ for(String jar : functionJars) {
+ udfURIs.add(parseURI(jar, false));
+ }
+ }
}
+
// create/drop function is allowed with any database
currDB = Database.ALL;
break;
http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-tests/data/xudf.jar
----------------------------------------------------------------------
diff --git a/sentry-tests/data/xudf.jar b/sentry-tests/data/xudf.jar
new file mode 100644
index 0000000..5bc4cee
Binary files /dev/null and b/sentry-tests/data/xudf.jar differ
http://git-wip-us.apache.org/repos/asf/sentry/blob/58d76b93/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
index 103a057..e440fb2 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java
@@ -17,9 +17,7 @@ printf_test_3 * Licensed to the Apache Software Foundation (ASF) under one or mo
package org.apache.sentry.tests.e2e.hive;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
import java.io.File;
import java.io.FileOutputStream;
@@ -86,6 +84,8 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu
String udf1ClassName = "org.apache.sentry.tests.e2e.hive.TestUDF";
CodeSource udf1Src = Class.forName(udf1ClassName).getProtectionDomain().getCodeSource();
String udf1Location = udf1Src.getLocation().getPath();
+ String udf2Location = new File("../data/xudf.jar").getCanonicalPath();
+
Connection connection = context.createConnection(ADMIN1);
Statement statement = context.createStatement(connection);
statement.execute("DROP DATABASE IF EXISTS " + DB1 + " CASCADE");
@@ -100,13 +100,14 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu
context.close();
policyFile
- .addRolesToGroup(USERGROUP1, "db1_all", "UDF1_JAR", "UDF_JAR", "data_read")
+ .addRolesToGroup(USERGROUP1, "db1_all", "UDF2_JAR", "UDF1_JAR", "UDF_JAR", "data_read")
.addRolesToGroup(USERGROUP2, "db1_tab1", "UDF_JAR")
.addRolesToGroup(USERGROUP3, "db1_tab1")
.addPermissionsToRole("db1_all", "server=server1->db=" + DB1)
.addPermissionsToRole("db1_tab1", "server=server1->db=" + DB1 + "->table=" + tableName1)
.addPermissionsToRole("UDF_JAR", "server=server1->uri=file://" + udfLocation)
.addPermissionsToRole("UDF1_JAR", "server=server1->uri=file://" + udf1Location)
+ .addPermissionsToRole("UDF2_JAR", "server=server1->uri=file://" + udf2Location)
.addPermissionsToRole("data_read", "server=server1->URI=" + "file:///tmp");
writePolicyFile(policyFile);
}
@@ -343,6 +344,57 @@ public class TestPrivilegesAtFunctionScope extends AbstractTestWithStaticConfigu
context.close();
}
+ /**
+ * Test create function using jar functionality
+ * @throws Exception
+ */
+ @Test
+ public void testAndVerifyFuncPrivilegesPart4() throws Exception {
+ setUpContext();
+ Connection connection = context.createConnection(USER1_1);
+ Statement statement = context.createStatement(connection);
+ statement.execute("USE " + DB1);
+
+ // USER1 has URI perm on jarFiles
+ try {
+ String udfLocation = new File("../data/xudf.jar").getCanonicalPath();
+
+ statement
+ .execute("CREATE FUNCTION "
+ + DB1
+ + ".xadd AS 'xudf.XAdd'"
+ + " using jar 'file://" + udfLocation + "'");
+ ResultSet rs = statement.executeQuery("select xadd(1)");
+ assertTrue(rs.next());
+ assertTrue(rs.getInt(1) == 1);
+ } catch (SQLException e) {
+ assertFalse("CREATE FUNCTION should succeed for user1:" + e, true);
+ } finally {
+ connection.close();
+ }
+
+ connection = context.createConnection(USER2_1);
+ statement = context.createStatement(connection);
+ statement.execute("USE " + DB1);
+
+ // USER2 doesn't have URI perm on jarFiles
+ try {
+ String udfLocation = new File("../data/xudf.jar").getCanonicalPath();
+
+ statement
+ .execute("CREATE FUNCTION "
+ + DB1
+ + ".xadd AS 'xudf.XAdd'"
+ + " using jar 'file://" + udfLocation + "'");
+ assertFalse("CREATE FUNCTION should fail for user2", true);
+ } catch (SQLException e) {
+ context.verifyAuthzException(e);
+ } finally {
+ connection.close();
+ }
+ context.close();
+ }
+
@Test
public void testUdfWhiteList () throws Exception {
String tableName1 = "tab1";